From e7145018ef1d56b061de0cc3a6a9ac83bc54f0f5 Mon Sep 17 00:00:00 2001 From: Ulric Qin Date: Wed, 23 Mar 2022 13:36:22 +0800 Subject: [PATCH] add alerts and dashboards --- docker/n9eetc/alerts/mysql_by_exporter.json | 302 ++++++++++++++++++ docker/n9eetc/alerts/redis_by_exporter.json | 212 ++++++++++++ .../n9eetc/dashboards/mysql_by_exporter.json | 119 +++++++ .../n9eetc/dashboards/redis_by_exporter.json | 77 +++++ docker/n9eetc/webapi.conf | 3 + etc/alerts/mysql_by_exporter.json | 302 ++++++++++++++++++ etc/alerts/redis_by_exporter.json | 212 ++++++++++++ etc/dashboards/mysql_by_exporter.json | 119 +++++++ etc/dashboards/redis_by_exporter.json | 77 +++++ etc/webapi.conf | 3 + 10 files changed, 1426 insertions(+) create mode 100644 docker/n9eetc/alerts/mysql_by_exporter.json create mode 100644 docker/n9eetc/alerts/redis_by_exporter.json create mode 100644 docker/n9eetc/dashboards/mysql_by_exporter.json create mode 100644 docker/n9eetc/dashboards/redis_by_exporter.json create mode 100644 etc/alerts/mysql_by_exporter.json create mode 100644 etc/alerts/redis_by_exporter.json create mode 100644 etc/dashboards/mysql_by_exporter.json create mode 100644 etc/dashboards/redis_by_exporter.json diff --git a/docker/n9eetc/alerts/mysql_by_exporter.json b/docker/n9eetc/alerts/mysql_by_exporter.json new file mode 100644 index 00000000..d894cda9 --- /dev/null +++ b/docker/n9eetc/alerts/mysql_by_exporter.json @@ -0,0 +1,302 @@ +[ + { + "name": "MysqlInnodbLogWaits", + "note": "MySQL innodb log writes stalling", + "severity": 2, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "rate(mysql_global_status_innodb_log_waits[15m]) > 10", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlInnodbLogWaits" + ] + }, + { + "name": "MysqlSlaveIoThreadNotRunning", + "note": "MySQL Slave IO thread not running", + "severity": 1, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlSlaveIoThreadNotRunning" + ] + }, + { + "name": "MysqlSlaveReplicationLag", + "note": "", + "severity": 1, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "mysql_slave_status_master_server_id > 0 and ON (instance) (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 30", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlSlaveReplicationLag" + ] + }, + { + "name": "MysqlSlaveSqlThreadNotRunning", + "note": "MySQL Slave SQL thread not running", + "severity": 1, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlSlaveSqlThreadNotRunning" + ] + }, + { + "name": "Mysql刚刚有重启，请注意", + "note": "MySQL has just been restarted, less than one minute ago", + "severity": 3, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "mysql_global_status_uptime < 60", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlRestarted" + ] + }, + { + "name": "Mysql实例挂了", + "note": "", + "severity": 1, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "mysql_up == 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlDown" + ] + }, + { + "name": "Mysql打开了很多文件句柄，请注意", + "note": "More than 80% of MySQL files open", + "severity": 2, + "disabled": 0, + "prom_for_duration": 120, + "prom_ql": "avg by (instance) (mysql_global_status_innodb_num_open_files) / avg by (instance)(mysql_global_variables_open_files_limit) * 100 > 80", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlHighOpenFiles" + ] + }, + { + "name": "Mysql最近一分钟有慢查询出现", + "note": "MySQL server mysql has some new slow query", + "severity": 2, + "disabled": 0, + "prom_for_duration": 120, + "prom_ql": "increase(mysql_global_status_slow_queries[1m]) > 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlSlowQueries" + ] + }, + { + "name": "Mysql有超过60%的连接是running状态", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 120, + "prom_ql": "avg by (instance) (mysql_global_status_threads_running) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 60", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlHighThreadsRunning" + ] + }, + { + "name": "Mysql连接数已超过80%", + "note": "More than 80% of MySQL connections are in use", + "severity": 2, + "disabled": 0, + "prom_for_duration": 120, + "prom_ql": "avg by (instance) (mysql_global_status_threads_connected) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 80", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlTooManyConnections" + ] + } + ] \ No newline at end of file diff --git a/docker/n9eetc/alerts/redis_by_exporter.json b/docker/n9eetc/alerts/redis_by_exporter.json new file mode 100644 index 00000000..8e78c112 --- /dev/null +++ b/docker/n9eetc/alerts/redis_by_exporter.json @@ -0,0 +1,212 @@ +[ + { + "name": "Redis内存使用率较高", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "redis_memory_max_bytes > 0 and (redis_memory_used_bytes / redis_memory_max_bytes) > 0.85", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisHighMemoryUsage" + ] + }, + { + "name": "Redis出现拒绝连接", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "(rate(redis_rejected_connections_total[5m])) > 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisRejectedConnHigh" + ] + }, + { + "name": "Redis刚刚有重启，请注意", + "note": "", + "severity": 3, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "redis_uptime_in_seconds < 600", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisLowUptime" + ] + }, + { + "name": "Redis客户端连接数过高", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "(redis_connected_clients / redis_config_maxclients) > 0.85", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisHighClientsUsage" + ] + }, + { + "name": "Redis延迟高", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "sum(rate(redis_commands_duration_seconds_total[5m])) / sum(rate(redis_commands_processed_total[5m])) > 0.25", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisHighResponseTime" + ] + }, + { + "name": "Redis较低的命中率", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "rate(redis_keyspace_hits_total[5m])\n/\n(rate(redis_keyspace_misses_total[5m]) + rate(redis_keyspace_hits_total[5m]))\n< 0.9", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisLowHitRatio" + ] + }, + { + "name": "Redis驱逐率较高", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "(sum(rate(redis_evicted_keys_total[5m])) / sum(redis_db_keys)) > 0.1", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisHighKeysEvictionRatio" + ] + } + ] \ No newline at end of file diff --git a/docker/n9eetc/dashboards/mysql_by_exporter.json b/docker/n9eetc/dashboards/mysql_by_exporter.json new file mode 100644 index 00000000..b76ef591 --- /dev/null +++ b/docker/n9eetc/dashboards/mysql_by_exporter.json @@ -0,0 +1,119 @@ +[ + { + "name": "MySQL Overview - 模板", + "tags": "Prometheus MySQL", + "configs": "{\"var\":[{\"name\":\"instance\",\"definition\":\"label_values(mysql_global_status_uptime, instance)\"}]}", + "chart_groups": [ + { + "name": "Basic Info", + "weight": 0, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"min(mysql_global_status_uptime{instance=~\\\"$instance\\\"})\"}],\"name\":\"MySQL Uptime\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"valueMappings\":[{\"type\":\"range\",\"match\":{\"to\":1800},\"result\":{\"color\":\"#ec7718\"}},{\"type\":\"range\",\"match\":{\"from\":1800},\"result\":{\"color\":\"#369603\"}}],\"standardOptions\":{\"util\":\"humantimeSeconds\"}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"rate(mysql_global_status_queries{instance=~\\\"$instance\\\"}[5m])\"}],\"name\":\"Current QPS\",\"description\":\"mysql_global_status_queries\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"valueMappings\":[{\"type\":\"range\",\"match\":{\"to\":100},\"result\":{\"color\":\"#05a31f\"}},{\"type\":\"range\",\"match\":{\"from\":100},\"result\":{\"color\":\"#ea3939\"}}],\"standardOptions\":{\"decimals\":2}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"avg(mysql_global_variables_innodb_buffer_pool_size{instance=~\\\"$instance\\\"})\"}],\"name\":\"InnoDB Buffer Pool\",\"description\":\"**InnoDB Buffer Pool Size**\\n\\nInnoDB maintains a storage area called the buffer pool for caching data and indexes in memory. Knowing how the InnoDB buffer pool works, and taking advantage of it to keep frequently accessed data in memory, is one of the most important aspects of MySQL tuning. The goal is to keep the working set in memory. In most cases, this should be between 60%-90% of available memory on a dedicated database host, but depends on many factors.\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"standardOptions\":{\"util\":\"bytesIEC\"}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":12,\"y\":0,\"i\":\"2\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(increase(mysql_global_status_table_locks_waited{instance=~\\\"$instance\\\"}[5m]))\"}],\"name\":\"Table Locks Waited(5min)\",\"description\":\"**Table Locks**\\n\\nMySQL takes a number of different locks for varying reasons. In this graph we see how many Table level locks MySQL has requested from the storage engine. In the case of InnoDB, many times the locks could actually be row locks as it only takes table level locks in a few specific cases.\\n\\nIt is most useful to compare Locks Immediate and Locks Waited. If Locks waited is rising, it means you have lock contention. Otherwise, Locks Immediate rising and falling is normal activity.\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"valueMappings\":[{\"type\":\"range\",\"match\":{\"from\":1},\"result\":{\"color\":\"#e70d0d\"}},{\"type\":\"range\",\"match\":{\"to\":1},\"result\":{\"color\":\"#53b503\"}}],\"standardOptions\":{}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":18,\"y\":0,\"i\":\"3\"}}", + "weight": 0 + } + ] + }, + { + "name": "Connections", + "weight": 1, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum(mysql_global_status_threads_connected{instance=~\\\"$instance\\\"})\",\"legend\":\"Connections\"},{\"expr\":\"sum(mysql_global_status_max_used_connections{instance=~\\\"$instance\\\"})\",\"legend\":\"Max Used Connections\"},{\"expr\":\"sum(mysql_global_variables_max_connections{instance=~\\\"$instance\\\"})\",\"legend\":\"Max Connections\"},{\"expr\":\"sum(rate(mysql_global_status_aborted_connects{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Aborted Connections\"}],\"name\":\"MySQL Connections\",\"description\":\"**Max Connections** \\n\\nMax Connections is the maximum permitted number of simultaneous client connections. By default, this is 151. Increasing this value increases the number of file descriptors that mysqld requires. If the required number of descriptors are not available, the server reduces the value of Max Connections.\\n\\nmysqld actually permits Max Connections + 1 clients to connect. The extra connection is reserved for use by accounts that have the SUPER privilege, such as root.\\n\\nMax Used Connections is the maximum number of connections that have been in use simultaneously since the server started.\\n\\nConnections is the number of connection attempts (successful or not) to the MySQL server.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(mysql_global_status_threads_connected{instance=~\\\"$instance\\\"})\",\"legend\":\"Threads Connected\"},{\"expr\":\"sum(mysql_global_status_threads_running{instance=~\\\"$instance\\\"})\",\"legend\":\"Threads Running\"}],\"name\":\"MySQL Client Thread Activity\",\"description\":\"Threads Connected is the number of open connections, while Threads Running is the number of threads not sleeping.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + } + ] + }, + { + "name": "Query Performance", + "weight": 2, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_created_tmp_tables{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Created Tmp Tables\"},{\"expr\":\"sum(rate(mysql_global_status_created_tmp_disk_tables{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Created Tmp Disk Tables\"},{\"expr\":\"sum(rate(mysql_global_status_created_tmp_files{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Created Tmp Files\"}],\"name\":\"MySQL Temporary Objects\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.64,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_select_full_join{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Full Join\"},{\"expr\":\"sum(rate(mysql_global_status_select_full_range_join{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Full Range Join\"},{\"expr\":\"sum(rate(mysql_global_status_select_range{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Range\"},{\"expr\":\"sum(rate(mysql_global_status_select_range_check{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Range Check\"},{\"expr\":\"sum(rate(mysql_global_status_select_scan{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Scan\"}],\"name\":\"MySQL Select Types\",\"description\":\"**MySQL Select Types**\\n\\nAs with most relational databases, selecting based on indexes is more efficient than scanning an entire table's data. Here we see the counters for selects not done with indexes.\\n\\n* ***Select Scan*** is how many queries caused full table scans, in which all the data in the table had to be read and either discarded or returned.\\n* ***Select Range*** is how many queries used a range scan, which means MySQL scanned all rows in a given range.\\n* ***Select Full Join*** is the number of joins that are not joined on an index, this is usually a huge performance hit.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"stack\":\"off\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.41},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_sort_rows{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Sort Rows\"},{\"expr\":\"sum(rate(mysql_global_status_sort_range{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Sort Range\"},{\"expr\":\"sum(rate(mysql_global_status_sort_merge_passes{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Sort Merge Passes\"},{\"expr\":\"sum(rate(mysql_global_status_sort_scan{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Sort Scan\"}],\"name\":\"MySQL Sorts\",\"description\":\"**MySQL Sorts**\\n\\nDue to a query's structure, order, or other requirements, MySQL sorts the rows before returning them. For example, if a table is ordered 1 to 10 but you want the results reversed, MySQL then has to sort the rows to return 10 to 1.\\n\\nThis graph also shows when sorts had to scan a whole table or a given range of a table in order to return the results and which could not have been sorted via an index.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":2,\"i\":\"2\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_slow_queries{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Slow Queries\"}],\"name\":\"MySQL Slow Queries\",\"description\":\"**MySQL Slow Queries**\\n\\nSlow queries are defined as queries being slower than the long_query_time setting. For example, if you have long_query_time set to 3, all queries that take longer than 3 seconds to complete will show on this graph.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"bars\",\"stack\":\"off\",\"fillOpacity\":0.81},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":2,\"i\":\"3\"}}", + "weight": 0 + } + ] + }, + { + "name": "Network", + "weight": 3, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_bytes_received{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Inbound\"},{\"expr\":\"sum(rate(mysql_global_status_bytes_sent{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Outbound\"}],\"name\":\"MySQL Network Traffic\",\"description\":\"**MySQL Network Traffic**\\n\\nHere we can see how much network traffic is generated by MySQL. Outbound is network traffic sent from MySQL and Inbound is network traffic MySQL has received.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"util\":\"bytesSI\",\"decimals\":2},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":24,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + } + ] + }, + { + "name": "Commands, Handlers", + "weight": 4, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"topk(10, rate(mysql_global_status_commands_total{instance=~\\\"$instance\\\"}[5m])>0)\",\"legend\":\"Com_{{command}}\"}],\"name\":\"Top Command Counters\",\"description\":\"**Top Command Counters**\\n\\nThe Com_{{xxx}} statement counter variables indicate the number of times each xxx statement has been executed. There is one status variable for each type of statement. For example, Com_delete and Com_update count [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements, respectively. Com_delete_multi and Com_update_multi are similar but apply to [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements that use multiple-table syntax.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"decimals\":2},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.2,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":24,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"rate(mysql_global_status_handlers_total{instance=~\\\"$instance\\\", handler!~\\\"commit|rollback|savepoint.*|prepare\\\"}[5m])\",\"legend\":\"{{handler}}\"}],\"name\":\"MySQL Handlers\",\"description\":\"**MySQL Handlers**\\n\\nHandler statistics are internal statistics on how MySQL is selecting, updating, inserting, and modifying rows, tables, and indexes.\\n\\nThis is in fact the layer between the Storage Engine and MySQL.\\n\\n* `read_rnd_next` is incremented when the server performs a full table scan and this is a counter you don't really want to see with a high value.\\n* `read_key` is incremented when a read is done with an index.\\n* `read_next` is incremented when the storage engine is asked to 'read the next index entry'. A high value means a lot of index scans are being done.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"decimals\":3},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":2,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"rate(mysql_global_status_handlers_total{instance=~\\\"$instance\\\", handler=~\\\"commit|rollback|savepoint.*|prepare\\\"}[5m])\",\"legend\":\"{{handler}}\"}],\"name\":\"MySQL Transaction Handlers\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":2,\"i\":\"2\"}}", + "weight": 0 + } + ] + }, + { + "name": "Open Files", + "weight": 5, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"mysql_global_variables_open_files_limit{instance=~\\\"$instance\\\"}\",\"legend\":\"Open Files Limit\"},{\"expr\":\"mysql_global_status_innodb_num_open_files{instance=~\\\"$instance\\\"}\",\"legend\":\"InnoDB Open Files\"}],\"name\":\"MySQL Open Files\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":24,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + } + ] + }, + { + "name": "Table Openings", + "weight": 6, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"rate(mysql_global_status_table_open_cache_hits{instance=~\\\"$instance\\\"}[5m])\\n/\\n(\\nrate(mysql_global_status_table_open_cache_hits{instance=~\\\"$instance\\\"}[5m])\\n+\\nrate(mysql_global_status_table_open_cache_misses{instance=~\\\"$instance\\\"}[5m])\\n)\",\"legend\":\"Table Open Cache Hit Ratio\"}],\"name\":\"Table Open Cache Hit Ratio\",\"description\":\"**MySQL Table Open Cache Status**\\n\\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\\n\\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"util\":\"percentUnit\"},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"mysql_global_status_open_tables{instance=~\\\"$instance\\\"}\",\"legend\":\"Open Tables\"},{\"expr\":\"mysql_global_variables_table_open_cache{instance=~\\\"$instance\\\"}\",\"legend\":\"Table Open Cache\"}],\"name\":\"MySQL Open Tables\",\"description\":\"**MySQL Open Tables**\\n\\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\\n\\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + } + ] + } + ] + } + ] \ No newline at end of file diff --git a/docker/n9eetc/dashboards/redis_by_exporter.json b/docker/n9eetc/dashboards/redis_by_exporter.json new file mode 100644 index 00000000..8e809019 --- /dev/null +++ b/docker/n9eetc/dashboards/redis_by_exporter.json @@ -0,0 +1,77 @@ +[ + { + "name": "Redis Overview - 模板", + "tags": "Redis Prometheus", + "configs": "{\"var\":[{\"name\":\"instance\",\"definition\":\"label_values(redis_uptime_in_seconds,instance)\",\"selected\":\"10.206.0.16:6379\"}]}", + "chart_groups": [ + { + "name": "Basic Info", + "weight": 0, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"min(redis_uptime_in_seconds{instance=~\\\"$instance\\\"})\"}],\"name\":\"Redis Uptime\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"standardOptions\":{\"util\":\"humantimeSeconds\"}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(redis_connected_clients{instance=~\\\"$instance\\\"})\"}],\"name\":\"Connected Clients\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"standardOptions\":{}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"redis_memory_used_bytes{instance=~\\\"$instance\\\"}\"}],\"name\":\"Memory Used\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"valueMappings\":[{\"type\":\"range\",\"match\":{\"to\":128000000},\"result\":{\"color\":\"#079e05\"}},{\"type\":\"range\",\"match\":{\"from\":128000000},\"result\":{\"color\":\"#f10909\"}}],\"standardOptions\":{\"util\":\"bytesIEC\",\"decimals\":0}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":12,\"y\":0,\"i\":\"2\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"redis_memory_max_bytes{instance=~\\\"$instance\\\"}\"}],\"name\":\"Max Memory Limit\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"standardOptions\":{\"util\":\"bytesIEC\"}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":18,\"y\":0,\"i\":\"3\"}}", + "weight": 0 + } + ] + }, + { + "name": "Commands", + "weight": 1, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"rate(redis_commands_processed_total{instance=~\\\"$instance\\\"}[5m])\"}],\"name\":\"Commands Executed / sec\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"irate(redis_keyspace_hits_total{instance=~\\\"$instance\\\"}[5m])\",\"legend\":\"hits\"},{\"expr\":\"irate(redis_keyspace_misses_total{instance=~\\\"$instance\\\"}[5m])\",\"legend\":\"misses\"}],\"name\":\"Hits / Misses per Sec\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"noraml\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":8,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"topk(5, irate(redis_commands_total{instance=~\\\"$instance\\\"} [1m]))\",\"legend\":\"{{cmd}}\"}],\"name\":\"Top Commands\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":16,\"y\":0,\"i\":\"2\"}}", + "weight": 0 + } + ] + }, + { + "name": "Keys", + "weight": 2, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum (redis_db_keys{instance=~\\\"$instance\\\"}) by (db)\",\"legend\":\"{{db}}\"}],\"name\":\"Total Items per DB\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(redis_expired_keys_total{instance=~\\\"$instance\\\"}[5m])) by (instance)\",\"legend\":\"expired\"},{\"expr\":\"sum(rate(redis_evicted_keys_total{instance=~\\\"$instance\\\"}[5m])) by (instance)\",\"legend\":\"evicted\"}],\"name\":\"Expired / Evicted\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":8,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(redis_db_keys{instance=~\\\"$instance\\\"}) - sum(redis_db_keys_expiring{instance=~\\\"$instance\\\"}) \",\"legend\":\"not expiring\"},{\"expr\":\"sum(redis_db_keys_expiring{instance=~\\\"$instance\\\"}) \",\"legend\":\"expiring\"}],\"name\":\"Expiring vs Not-Expiring Keys\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"noraml\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":16,\"y\":0,\"i\":\"2\"}}", + "weight": 0 + } + ] + }, + { + "name": "Network", + "weight": 3, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(redis_net_input_bytes_total{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"input\"},{\"expr\":\"sum(rate(redis_net_output_bytes_total{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"output\"}],\"name\":\"Network I/O\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"util\":\"bytesIEC\",\"decimals\":2},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":24,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + } + ] + } + ] + } + ] \ No newline at end of file diff --git a/docker/n9eetc/webapi.conf b/docker/n9eetc/webapi.conf index de55ff5b..4240cf4e 100644 --- a/docker/n9eetc/webapi.conf +++ b/docker/n9eetc/webapi.conf @@ -7,6 +7,9 @@ RunMode = "release" # metrics descriptions MetricsYamlFile = "./etc/metrics.yaml" +BuiltinAlertsDir = "./etc/alerts" +BuiltinDashboardsDir = "./etc/dashboards" + [[NotifyChannels]] Label = "邮箱" # do not change Key diff --git a/etc/alerts/mysql_by_exporter.json b/etc/alerts/mysql_by_exporter.json new file mode 100644 index 00000000..d894cda9 --- /dev/null +++ b/etc/alerts/mysql_by_exporter.json @@ -0,0 +1,302 @@ +[ + { + "name": "MysqlInnodbLogWaits", + "note": "MySQL innodb log writes stalling", + "severity": 2, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "rate(mysql_global_status_innodb_log_waits[15m]) > 10", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlInnodbLogWaits" + ] + }, + { + "name": "MysqlSlaveIoThreadNotRunning", + "note": "MySQL Slave IO thread not running", + "severity": 1, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlSlaveIoThreadNotRunning" + ] + }, + { + "name": "MysqlSlaveReplicationLag", + "note": "", + "severity": 1, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "mysql_slave_status_master_server_id > 0 and ON (instance) (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 30", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlSlaveReplicationLag" + ] + }, + { + "name": "MysqlSlaveSqlThreadNotRunning", + "note": "MySQL Slave SQL thread not running", + "severity": 1, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlSlaveSqlThreadNotRunning" + ] + }, + { + "name": "Mysql刚刚有重启，请注意", + "note": "MySQL has just been restarted, less than one minute ago", + "severity": 3, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "mysql_global_status_uptime < 60", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlRestarted" + ] + }, + { + "name": "Mysql实例挂了", + "note": "", + "severity": 1, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "mysql_up == 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlDown" + ] + }, + { + "name": "Mysql打开了很多文件句柄，请注意", + "note": "More than 80% of MySQL files open", + "severity": 2, + "disabled": 0, + "prom_for_duration": 120, + "prom_ql": "avg by (instance) (mysql_global_status_innodb_num_open_files) / avg by (instance)(mysql_global_variables_open_files_limit) * 100 > 80", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlHighOpenFiles" + ] + }, + { + "name": "Mysql最近一分钟有慢查询出现", + "note": "MySQL server mysql has some new slow query", + "severity": 2, + "disabled": 0, + "prom_for_duration": 120, + "prom_ql": "increase(mysql_global_status_slow_queries[1m]) > 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlSlowQueries" + ] + }, + { + "name": "Mysql有超过60%的连接是running状态", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 120, + "prom_ql": "avg by (instance) (mysql_global_status_threads_running) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 60", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlHighThreadsRunning" + ] + }, + { + "name": "Mysql连接数已超过80%", + "note": "More than 80% of MySQL connections are in use", + "severity": 2, + "disabled": 0, + "prom_for_duration": 120, + "prom_ql": "avg by (instance) (mysql_global_status_threads_connected) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 80", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=MysqlTooManyConnections" + ] + } + ] \ No newline at end of file diff --git a/etc/alerts/redis_by_exporter.json b/etc/alerts/redis_by_exporter.json new file mode 100644 index 00000000..8e78c112 --- /dev/null +++ b/etc/alerts/redis_by_exporter.json @@ -0,0 +1,212 @@ +[ + { + "name": "Redis内存使用率较高", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "redis_memory_max_bytes > 0 and (redis_memory_used_bytes / redis_memory_max_bytes) > 0.85", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisHighMemoryUsage" + ] + }, + { + "name": "Redis出现拒绝连接", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "(rate(redis_rejected_connections_total[5m])) > 0", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisRejectedConnHigh" + ] + }, + { + "name": "Redis刚刚有重启，请注意", + "note": "", + "severity": 3, + "disabled": 0, + "prom_for_duration": 0, + "prom_ql": "redis_uptime_in_seconds < 600", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisLowUptime" + ] + }, + { + "name": "Redis客户端连接数过高", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "(redis_connected_clients / redis_config_maxclients) > 0.85", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisHighClientsUsage" + ] + }, + { + "name": "Redis延迟高", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "sum(rate(redis_commands_duration_seconds_total[5m])) / sum(rate(redis_commands_processed_total[5m])) > 0.25", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisHighResponseTime" + ] + }, + { + "name": "Redis较低的命中率", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "rate(redis_keyspace_hits_total[5m])\n/\n(rate(redis_keyspace_misses_total[5m]) + rate(redis_keyspace_hits_total[5m]))\n< 0.9", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisLowHitRatio" + ] + }, + { + "name": "Redis驱逐率较高", + "note": "", + "severity": 2, + "disabled": 0, + "prom_for_duration": 60, + "prom_ql": "(sum(rate(redis_evicted_keys_total[5m])) / sum(redis_db_keys)) > 0.1", + "prom_eval_interval": 15, + "enable_stime": "00:00", + "enable_etime": "23:59", + "enable_days_of_week": [ + "1", + "2", + "3", + "4", + "5", + "6", + "0" + ], + "enable_in_bg": 0, + "notify_recovered": 1, + "notify_channels": [], + "notify_repeat_step": 60, + "recover_duration": 0, + "callbacks": [], + "runbook_url": "", + "append_tags": [ + "alertname=RedisHighKeysEvictionRatio" + ] + } + ] \ No newline at end of file diff --git a/etc/dashboards/mysql_by_exporter.json b/etc/dashboards/mysql_by_exporter.json new file mode 100644 index 00000000..b76ef591 --- /dev/null +++ b/etc/dashboards/mysql_by_exporter.json @@ -0,0 +1,119 @@ +[ + { + "name": "MySQL Overview - 模板", + "tags": "Prometheus MySQL", + "configs": "{\"var\":[{\"name\":\"instance\",\"definition\":\"label_values(mysql_global_status_uptime, instance)\"}]}", + "chart_groups": [ + { + "name": "Basic Info", + "weight": 0, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"min(mysql_global_status_uptime{instance=~\\\"$instance\\\"})\"}],\"name\":\"MySQL Uptime\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"valueMappings\":[{\"type\":\"range\",\"match\":{\"to\":1800},\"result\":{\"color\":\"#ec7718\"}},{\"type\":\"range\",\"match\":{\"from\":1800},\"result\":{\"color\":\"#369603\"}}],\"standardOptions\":{\"util\":\"humantimeSeconds\"}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"rate(mysql_global_status_queries{instance=~\\\"$instance\\\"}[5m])\"}],\"name\":\"Current QPS\",\"description\":\"mysql_global_status_queries\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"valueMappings\":[{\"type\":\"range\",\"match\":{\"to\":100},\"result\":{\"color\":\"#05a31f\"}},{\"type\":\"range\",\"match\":{\"from\":100},\"result\":{\"color\":\"#ea3939\"}}],\"standardOptions\":{\"decimals\":2}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"avg(mysql_global_variables_innodb_buffer_pool_size{instance=~\\\"$instance\\\"})\"}],\"name\":\"InnoDB Buffer Pool\",\"description\":\"**InnoDB Buffer Pool Size**\\n\\nInnoDB maintains a storage area called the buffer pool for caching data and indexes in memory. Knowing how the InnoDB buffer pool works, and taking advantage of it to keep frequently accessed data in memory, is one of the most important aspects of MySQL tuning. The goal is to keep the working set in memory. In most cases, this should be between 60%-90% of available memory on a dedicated database host, but depends on many factors.\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"standardOptions\":{\"util\":\"bytesIEC\"}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":12,\"y\":0,\"i\":\"2\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(increase(mysql_global_status_table_locks_waited{instance=~\\\"$instance\\\"}[5m]))\"}],\"name\":\"Table Locks Waited(5min)\",\"description\":\"**Table Locks**\\n\\nMySQL takes a number of different locks for varying reasons. In this graph we see how many Table level locks MySQL has requested from the storage engine. In the case of InnoDB, many times the locks could actually be row locks as it only takes table level locks in a few specific cases.\\n\\nIt is most useful to compare Locks Immediate and Locks Waited. If Locks waited is rising, it means you have lock contention. Otherwise, Locks Immediate rising and falling is normal activity.\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"valueMappings\":[{\"type\":\"range\",\"match\":{\"from\":1},\"result\":{\"color\":\"#e70d0d\"}},{\"type\":\"range\",\"match\":{\"to\":1},\"result\":{\"color\":\"#53b503\"}}],\"standardOptions\":{}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":18,\"y\":0,\"i\":\"3\"}}", + "weight": 0 + } + ] + }, + { + "name": "Connections", + "weight": 1, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum(mysql_global_status_threads_connected{instance=~\\\"$instance\\\"})\",\"legend\":\"Connections\"},{\"expr\":\"sum(mysql_global_status_max_used_connections{instance=~\\\"$instance\\\"})\",\"legend\":\"Max Used Connections\"},{\"expr\":\"sum(mysql_global_variables_max_connections{instance=~\\\"$instance\\\"})\",\"legend\":\"Max Connections\"},{\"expr\":\"sum(rate(mysql_global_status_aborted_connects{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Aborted Connections\"}],\"name\":\"MySQL Connections\",\"description\":\"**Max Connections** \\n\\nMax Connections is the maximum permitted number of simultaneous client connections. By default, this is 151. Increasing this value increases the number of file descriptors that mysqld requires. If the required number of descriptors are not available, the server reduces the value of Max Connections.\\n\\nmysqld actually permits Max Connections + 1 clients to connect. The extra connection is reserved for use by accounts that have the SUPER privilege, such as root.\\n\\nMax Used Connections is the maximum number of connections that have been in use simultaneously since the server started.\\n\\nConnections is the number of connection attempts (successful or not) to the MySQL server.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(mysql_global_status_threads_connected{instance=~\\\"$instance\\\"})\",\"legend\":\"Threads Connected\"},{\"expr\":\"sum(mysql_global_status_threads_running{instance=~\\\"$instance\\\"})\",\"legend\":\"Threads Running\"}],\"name\":\"MySQL Client Thread Activity\",\"description\":\"Threads Connected is the number of open connections, while Threads Running is the number of threads not sleeping.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + } + ] + }, + { + "name": "Query Performance", + "weight": 2, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_created_tmp_tables{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Created Tmp Tables\"},{\"expr\":\"sum(rate(mysql_global_status_created_tmp_disk_tables{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Created Tmp Disk Tables\"},{\"expr\":\"sum(rate(mysql_global_status_created_tmp_files{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Created Tmp Files\"}],\"name\":\"MySQL Temporary Objects\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.64,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_select_full_join{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Full Join\"},{\"expr\":\"sum(rate(mysql_global_status_select_full_range_join{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Full Range Join\"},{\"expr\":\"sum(rate(mysql_global_status_select_range{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Range\"},{\"expr\":\"sum(rate(mysql_global_status_select_range_check{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Range Check\"},{\"expr\":\"sum(rate(mysql_global_status_select_scan{ instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Select Scan\"}],\"name\":\"MySQL Select Types\",\"description\":\"**MySQL Select Types**\\n\\nAs with most relational databases, selecting based on indexes is more efficient than scanning an entire table's data. Here we see the counters for selects not done with indexes.\\n\\n* ***Select Scan*** is how many queries caused full table scans, in which all the data in the table had to be read and either discarded or returned.\\n* ***Select Range*** is how many queries used a range scan, which means MySQL scanned all rows in a given range.\\n* ***Select Full Join*** is the number of joins that are not joined on an index, this is usually a huge performance hit.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"stack\":\"off\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.41},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_sort_rows{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Sort Rows\"},{\"expr\":\"sum(rate(mysql_global_status_sort_range{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Sort Range\"},{\"expr\":\"sum(rate(mysql_global_status_sort_merge_passes{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Sort Merge Passes\"},{\"expr\":\"sum(rate(mysql_global_status_sort_scan{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Sort Scan\"}],\"name\":\"MySQL Sorts\",\"description\":\"**MySQL Sorts**\\n\\nDue to a query's structure, order, or other requirements, MySQL sorts the rows before returning them. For example, if a table is ordered 1 to 10 but you want the results reversed, MySQL then has to sort the rows to return 10 to 1.\\n\\nThis graph also shows when sorts had to scan a whole table or a given range of a table in order to return the results and which could not have been sorted via an index.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":2,\"i\":\"2\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_slow_queries{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Slow Queries\"}],\"name\":\"MySQL Slow Queries\",\"description\":\"**MySQL Slow Queries**\\n\\nSlow queries are defined as queries being slower than the long_query_time setting. For example, if you have long_query_time set to 3, all queries that take longer than 3 seconds to complete will show on this graph.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"bars\",\"stack\":\"off\",\"fillOpacity\":0.81},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":2,\"i\":\"3\"}}", + "weight": 0 + } + ] + }, + { + "name": "Network", + "weight": 3, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(mysql_global_status_bytes_received{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Inbound\"},{\"expr\":\"sum(rate(mysql_global_status_bytes_sent{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"Outbound\"}],\"name\":\"MySQL Network Traffic\",\"description\":\"**MySQL Network Traffic**\\n\\nHere we can see how much network traffic is generated by MySQL. Outbound is network traffic sent from MySQL and Inbound is network traffic MySQL has received.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"util\":\"bytesSI\",\"decimals\":2},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":24,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + } + ] + }, + { + "name": "Commands, Handlers", + "weight": 4, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"topk(10, rate(mysql_global_status_commands_total{instance=~\\\"$instance\\\"}[5m])>0)\",\"legend\":\"Com_{{command}}\"}],\"name\":\"Top Command Counters\",\"description\":\"**Top Command Counters**\\n\\nThe Com_{{xxx}} statement counter variables indicate the number of times each xxx statement has been executed. There is one status variable for each type of statement. For example, Com_delete and Com_update count [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements, respectively. Com_delete_multi and Com_update_multi are similar but apply to [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements that use multiple-table syntax.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"decimals\":2},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.2,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":24,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"rate(mysql_global_status_handlers_total{instance=~\\\"$instance\\\", handler!~\\\"commit|rollback|savepoint.*|prepare\\\"}[5m])\",\"legend\":\"{{handler}}\"}],\"name\":\"MySQL Handlers\",\"description\":\"**MySQL Handlers**\\n\\nHandler statistics are internal statistics on how MySQL is selecting, updating, inserting, and modifying rows, tables, and indexes.\\n\\nThis is in fact the layer between the Storage Engine and MySQL.\\n\\n* `read_rnd_next` is incremented when the server performs a full table scan and this is a counter you don't really want to see with a high value.\\n* `read_key` is incremented when a read is done with an index.\\n* `read_next` is incremented when the storage engine is asked to 'read the next index entry'. A high value means a lot of index scans are being done.\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"decimals\":3},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":2,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"rate(mysql_global_status_handlers_total{instance=~\\\"$instance\\\", handler=~\\\"commit|rollback|savepoint.*|prepare\\\"}[5m])\",\"legend\":\"{{handler}}\"}],\"name\":\"MySQL Transaction Handlers\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":2,\"i\":\"2\"}}", + "weight": 0 + } + ] + }, + { + "name": "Open Files", + "weight": 5, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"mysql_global_variables_open_files_limit{instance=~\\\"$instance\\\"}\",\"legend\":\"Open Files Limit\"},{\"expr\":\"mysql_global_status_innodb_num_open_files{instance=~\\\"$instance\\\"}\",\"legend\":\"InnoDB Open Files\"}],\"name\":\"MySQL Open Files\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":24,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + } + ] + }, + { + "name": "Table Openings", + "weight": 6, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"rate(mysql_global_status_table_open_cache_hits{instance=~\\\"$instance\\\"}[5m])\\n/\\n(\\nrate(mysql_global_status_table_open_cache_hits{instance=~\\\"$instance\\\"}[5m])\\n+\\nrate(mysql_global_status_table_open_cache_misses{instance=~\\\"$instance\\\"}[5m])\\n)\",\"legend\":\"Table Open Cache Hit Ratio\"}],\"name\":\"Table Open Cache Hit Ratio\",\"description\":\"**MySQL Table Open Cache Status**\\n\\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\\n\\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"util\":\"percentUnit\"},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"mysql_global_status_open_tables{instance=~\\\"$instance\\\"}\",\"legend\":\"Open Tables\"},{\"expr\":\"mysql_global_variables_table_open_cache{instance=~\\\"$instance\\\"}\",\"legend\":\"Table Open Cache\"}],\"name\":\"MySQL Open Tables\",\"description\":\"**MySQL Open Tables**\\n\\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\\n\\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":12,\"x\":12,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + } + ] + } + ] + } + ] \ No newline at end of file diff --git a/etc/dashboards/redis_by_exporter.json b/etc/dashboards/redis_by_exporter.json new file mode 100644 index 00000000..8e809019 --- /dev/null +++ b/etc/dashboards/redis_by_exporter.json @@ -0,0 +1,77 @@ +[ + { + "name": "Redis Overview - 模板", + "tags": "Redis Prometheus", + "configs": "{\"var\":[{\"name\":\"instance\",\"definition\":\"label_values(redis_uptime_in_seconds,instance)\",\"selected\":\"10.206.0.16:6379\"}]}", + "chart_groups": [ + { + "name": "Basic Info", + "weight": 0, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"min(redis_uptime_in_seconds{instance=~\\\"$instance\\\"})\"}],\"name\":\"Redis Uptime\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"standardOptions\":{\"util\":\"humantimeSeconds\"}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(redis_connected_clients{instance=~\\\"$instance\\\"})\"}],\"name\":\"Connected Clients\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"standardOptions\":{}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"redis_memory_used_bytes{instance=~\\\"$instance\\\"}\"}],\"name\":\"Memory Used\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"valueMappings\":[{\"type\":\"range\",\"match\":{\"to\":128000000},\"result\":{\"color\":\"#079e05\"}},{\"type\":\"range\",\"match\":{\"from\":128000000},\"result\":{\"color\":\"#f10909\"}}],\"standardOptions\":{\"util\":\"bytesIEC\",\"decimals\":0}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":12,\"y\":0,\"i\":\"2\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"redis_memory_max_bytes{instance=~\\\"$instance\\\"}\"}],\"name\":\"Max Memory Limit\",\"custom\":{\"textMode\":\"value\",\"colorMode\":\"value\",\"calc\":\"lastNotNull\",\"colSpan\":1,\"textSize\":{}},\"options\":{\"standardOptions\":{\"util\":\"bytesIEC\"}},\"version\":\"2.0.0\",\"type\":\"stat\",\"layout\":{\"h\":1,\"w\":6,\"x\":18,\"y\":0,\"i\":\"3\"}}", + "weight": 0 + } + ] + }, + { + "name": "Commands", + "weight": 1, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"rate(redis_commands_processed_total{instance=~\\\"$instance\\\"}[5m])\"}],\"name\":\"Commands Executed / sec\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"irate(redis_keyspace_hits_total{instance=~\\\"$instance\\\"}[5m])\",\"legend\":\"hits\"},{\"expr\":\"irate(redis_keyspace_misses_total{instance=~\\\"$instance\\\"}[5m])\",\"legend\":\"misses\"}],\"name\":\"Hits / Misses per Sec\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"noraml\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":8,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"topk(5, irate(redis_commands_total{instance=~\\\"$instance\\\"} [1m]))\",\"legend\":\"{{cmd}}\"}],\"name\":\"Top Commands\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":16,\"y\":0,\"i\":\"2\"}}", + "weight": 0 + } + ] + }, + { + "name": "Keys", + "weight": 2, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum (redis_db_keys{instance=~\\\"$instance\\\"}) by (db)\",\"legend\":\"{{db}}\"}],\"name\":\"Total Items per DB\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(redis_expired_keys_total{instance=~\\\"$instance\\\"}[5m])) by (instance)\",\"legend\":\"expired\"},{\"expr\":\"sum(rate(redis_evicted_keys_total{instance=~\\\"$instance\\\"}[5m])) by (instance)\",\"legend\":\"evicted\"}],\"name\":\"Expired / Evicted\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":8,\"y\":0,\"i\":\"1\"}}", + "weight": 0 + }, + { + "configs": "{\"targets\":[{\"expr\":\"sum(redis_db_keys{instance=~\\\"$instance\\\"}) - sum(redis_db_keys_expiring{instance=~\\\"$instance\\\"}) \",\"legend\":\"not expiring\"},{\"expr\":\"sum(redis_db_keys_expiring{instance=~\\\"$instance\\\"}) \",\"legend\":\"expiring\"}],\"name\":\"Expiring vs Not-Expiring Keys\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"none\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"noraml\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":8,\"x\":16,\"y\":0,\"i\":\"2\"}}", + "weight": 0 + } + ] + }, + { + "name": "Network", + "weight": 3, + "charts": [ + { + "configs": "{\"targets\":[{\"expr\":\"sum(rate(redis_net_input_bytes_total{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"input\"},{\"expr\":\"sum(rate(redis_net_output_bytes_total{instance=~\\\"$instance\\\"}[5m]))\",\"legend\":\"output\"}],\"name\":\"Network I/O\",\"options\":{\"tooltip\":{\"mode\":\"all\",\"sort\":\"desc\"},\"legend\":{\"displayMode\":\"hidden\"},\"standardOptions\":{\"util\":\"bytesIEC\",\"decimals\":2},\"thresholds\":{}},\"custom\":{\"drawStyle\":\"lines\",\"lineInterpolation\":\"smooth\",\"fillOpacity\":0.5,\"stack\":\"off\"},\"version\":\"2.0.0\",\"type\":\"timeseries\",\"layout\":{\"h\":2,\"w\":24,\"x\":0,\"y\":0,\"i\":\"0\"}}", + "weight": 0 + } + ] + } + ] + } + ] \ No newline at end of file diff --git a/etc/webapi.conf b/etc/webapi.conf index bf2997fe..42c499c2 100644 --- a/etc/webapi.conf +++ b/etc/webapi.conf @@ -7,6 +7,9 @@ RunMode = "release" # metrics descriptions MetricsYamlFile = "./etc/metrics.yaml" +BuiltinAlertsDir = "./etc/alerts" +BuiltinDashboardsDir = "./etc/dashboards" + [[NotifyChannels]] Label = "邮箱" # do not change Key