innov
/
nightingale
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

新增blackbox_exporter支持 (#740) 

* 1. notify.py 支持安装channel反射发送
2. 支持钉钉群发送
3. 生成告警模板信息

* 1. notify.py 支持安装channel反射发送
2. 支持钉钉群发送
3. 增加二开说明

* 1. notify.py 用户创建一个虚拟的用户保存上述im群 的机器人token信息 user的contacts map中

* 1. notify.py alerts目录改为原来的

* 1. notify.py dingtalk send continue匹配

* 1. push型告警支持多条件 任意一个触发就触发

* 1. prometheus查询接口 tag-keys tag-values支持 params为空的情况

* 1. prometheus查询接口 ident匹配全部改为精确匹配
2. tagKey 提示改为tag_key

* 1. prometheus查询接口 支持instance_query 对外暴露

* 1. prometheus instance_query改名为instant-query
2. page group中去掉数据查询相关path

* 1. prometheus range_query 时间戳改为秒级
2. 查询支持传入分辨率参数

* 1. 新增jmx_exporter内置大盘

* 1. 新增blackbox_exporter内置大盘
2. 新增blackbox_exporter内置告警策略
This commit is contained in:
ning1875 2021-07-22 15:23:49 +08:00 committed by GitHub
parent 3236d7dfd1
commit 2d15445482
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 417 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
etc/alert_rule/blackbox_exporter Normal file
View File

@ -0,0 +1,191 @@
[
{
"name": "dns解析时间超过2秒",
"type": 1,
"expression": {
"evaluation_interval": 10,
"promql": "probe_dns_lookup_time_seconds>2"
},
"status": 0,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": "1 2 3 4 5 6 7",
"recovery_notify": 0,
"priority": 3,
"notify_channels": "",
"runbook_url": "",
"note": "",
"create_at": 1626935980,
"alert_duration": 60,
"notify_users_detail": null,
"notify_groups_detail": null
},
{
"name": "https证书过期时间小于7天",
"type": 1,
"expression": {
"evaluation_interval": 10,
"promql": "(probe_ssl_earliest_cert_expiry - time()) / 3600 / 24 <7"
},
"status": 0,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": "1 2 3 4 5 6 7",
"recovery_notify": 0,
"priority": 1,
"notify_channels": "",
"runbook_url": "",
"note": "",
"create_at": 1626935909,
"alert_duration": 60,
"notify_users_detail": null,
"notify_groups_detail": null
},
{
"name": "http响应数据传输占比超过70%",
"type": 1,
"expression": {
"evaluation_interval": 10,
"promql": "100 * avg(probe_http_duration_seconds{phase=\"transfer\"})by(instance) / sum(probe_http_duration_seconds) by(instance) >70"
},
"status": 0,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": "1 2 3 4 5 6 7",
"recovery_notify": 0,
"priority": 2,
"notify_channels": "",
"runbook_url": "",
"note": "",
"create_at": 1626936324,
"alert_duration": 60,
"notify_users_detail": null,
"notify_groups_detail": null
},
{
"name": "http接口探测失败",
"type": 1,
"expression": {
"evaluation_interval": 10,
"promql": "probe_success{job=~\".*http.*\"}==0"
},
"status": 0,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": "1 2 3 4 5 6 7",
"recovery_notify": 0,
"priority": 1,
"notify_channels": "",
"runbook_url": "",
"note": "",
"create_at": 1626935627,
"alert_duration": 60,
"notify_users_detail": null,
"notify_groups_detail": null
},
{
"name": "http接口探测耗时超过3秒",
"type": 1,
"expression": {
"evaluation_interval": 10,
"promql": "sum(probe_http_duration_seconds) by (instance) >3\n"
},
"status": 0,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": "1 2 3 4 5 6 7",
"recovery_notify": 0,
"priority": 3,
"notify_channels": "",
"runbook_url": "",
"note": "",
"create_at": 1626936059,
"alert_duration": 60,
"notify_users_detail": null,
"notify_groups_detail": null
},
{
"name": "http接口返回状态码4xx/5xx错误",
"type": 1,
"expression": {
"evaluation_interval": 10,
"promql": "probe_http_status_code >=400"
},
"status": 0,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": "1 2 3 4 5 6 7",
"recovery_notify": 0,
"priority": 1,
"notify_channels": "",
"runbook_url": "",
"note": "",
"create_at": 1626936145,
"alert_duration": 60,
"notify_users_detail": null,
"notify_groups_detail": null
},
{
"name": "icmp探测失败",
"type": 1,
"expression": {
"evaluation_interval": 10,
"promql": "probe_success{job=~\".*icmp.*\"}==0"
},
"status": 0,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": "1 2 3 4 5 6 7",
"recovery_notify": 0,
"priority": 1,
"notify_channels": "",
"runbook_url": "",
"note": "",
"create_at": 1626935855,
"alert_duration": 60,
"notify_users_detail": null,
"notify_groups_detail": null
},
{
"name": "tcp端口探测失败",
"type": 1,
"expression": {
"evaluation_interval": 10,
"promql": "probe_success{job=~\".*tcp.*\"}==0"
},
"status": 0,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": "1 2 3 4 5 6 7",
"recovery_notify": 0,
"priority": 1,
"notify_channels": "",
"runbook_url": "",
"note": "",
"create_at": 1626935874,
"alert_duration": 60,
"notify_users_detail": null,
"notify_groups_detail": null
},
{
"name": "机器ssh探测失败",
"type": 1,
"expression": {
"evaluation_interval": 10,
"promql": "probe_success{job=~\".*ssh.*\"}==0\n"
},
"status": 0,
"enable_stime": "00:00",
"enable_etime": "23:59",
"enable_days_of_week": "1 2 3 4 5 6 7",
"recovery_notify": 0,
"priority": 1,
"notify_channels": "",
"runbook_url": "",
"note": "",
"create_at": 1626935827,
"alert_duration": 60,
"notify_users_detail": null,
"notify_groups_detail": null
}
]

226
etc/dashboard/blackbox_exporter Normal file
View File

@ -0,0 +1,226 @@
[
{
"id": 0,
"name": "blackbox_exporter",
"tags": "",
"configs": "{\"tags\":[{\"tagName\":\"http_probe_job\",\"key\":\"job\",\"value\":\"blackbox-http\",\"prefix\":false},{\"tagName\":\"http_probe_instance\",\"key\":\"instance\",\"value\":\"*\",\"prefix\":false}]}",
"chart_groups": [
{
"id": 0,
"dashboard_id": 0,
"name": "http接口探测",
"weight": 0,
"charts": [
{
"id": 440,
"group_id": 109,
"configs": "{\"name\":\"https的探测\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_http_ssl==1\"],\"layout\":{\"h\":2,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}",
"weight": 0
},
{
"id": 441,
"group_id": 109,
"configs": "{\"name\":\"http的探测\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_http_ssl==0\"],\"layout\":{\"h\":2,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}",
"weight": 0
},
{
"id": 442,
"group_id": 109,
"configs": "{\"name\":\"https探测目标个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_http_ssl==1)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":12,\"y\":0,\"i\":\"2\"}}",
"weight": 0
},
{
"id": 443,
"group_id": 109,
"configs": "{\"name\":\"http探测目标个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_http_ssl==0)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":18,\"y\":0,\"i\":\"3\"}}",
"weight": 0
},
{
"id": 446,
"group_id": 109,
"configs": "{\"name\":\"http探测成功个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_success{job=~\\\".*http.*\\\"}==1)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":6,\"y\":2,\"i\":\"4\"}}",
"weight": 0
},
{
"id": 447,
"group_id": 109,
"configs": "{\"name\":\"http探测失败列表\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_success{job=~\\\".*http.*\\\"}==0\"],\"layout\":{\"h\":2,\"w\":6,\"x\":12,\"y\":2,\"i\":\"5\"}}",
"weight": 0
},
{
"id": 448,
"group_id": 109,
"configs": "{\"name\":\"http探测失败个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_success{job=~\\\".*http.*\\\"}==0)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":0,\"y\":2,\"i\":\"6\"}}",
"weight": 0
},
{
"id": 449,
"group_id": 109,
"configs": "{\"name\":\"http探测总耗时 单位秒\",\"mode\":\"promethues\",\"prome_ql\":[\"sum(probe_http_duration_seconds) by (instance)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":18,\"y\":2,\"i\":\"7\"}}",
"weight": 0
}
]
},
{
"id": 0,
"dashboard_id": 0,
"name": "https接口探测汇总",
"weight": 1,
"charts": [
{
"id": 444,
"group_id": 110,
"configs": "{\"name\":\"tls版本信息\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_tls_version_info\"],\"layout\":{\"h\":2,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}",
"weight": 0
},
{
"id": 445,
"group_id": 110,
"configs": "{\"name\":\"tls证书过期时间 单位:天\",\"mode\":\"promethues\",\"prome_ql\":[\"(probe_ssl_earliest_cert_expiry - time()) / 3600 / 24\"],\"layout\":{\"h\":2,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}",
"weight": 0
}
]
},
{
"id": 0,
"dashboard_id": 0,
"name": "http接口各阶段耗时详情",
"weight": 2,
"charts": [
{
"id": 450,
"group_id": 111,
"configs": "{\"name\":\"单个目标的各阶段耗时\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_http_duration_seconds{instance=~\\\"$instance\\\"}\"],\"layout\":{\"h\":2,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}",
"weight": 0
},
{
"id": 451,
"group_id": 111,
"configs": "{\"name\":\"[阶段1] dns解析时间\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_http_duration_seconds{instance=~\\\"$instance\\\",phase=\\\"resolve\\\"}\"],\"layout\":{\"h\":2,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}",
"weight": 0
},
{
"id": 452,
"group_id": 111,
"configs": "{\"name\":\"[可无]tls握手时间\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_http_duration_seconds{instance=~\\\"$instance\\\",phase=\\\"tls\\\"}\"],\"layout\":{\"h\":2,\"w\":6,\"x\":12,\"y\":0,\"i\":\"2\"}}",
"weight": 0
},
{
"id": 453,
"group_id": 111,
"configs": "{\"name\":\"[阶段2] tcp连接耗时\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_http_duration_seconds{instance=~\\\"$instance\\\",phase=\\\"connect\\\"}\"],\"layout\":{\"h\":2,\"w\":6,\"x\":18,\"y\":0,\"i\":\"3\"}}",
"weight": 0
},
{
"id": 454,
"group_id": 111,
"configs": "{\"name\":\"[阶段3] 服务端处理耗时\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_http_duration_seconds{instance=~\\\"$instance\\\",phase=\\\"processing\\\"}\"],\"layout\":{\"h\":2,\"w\":6,\"x\":0,\"y\":2,\"i\":\"4\"}}",
"weight": 0
},
{
"id": 455,
"group_id": 111,
"configs": "{\"name\":\"[阶段4] 传输响应耗时\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_http_duration_seconds{instance=~\\\"$instance\\\",phase=\\\"transfer\\\"}\"],\"layout\":{\"h\":2,\"w\":6,\"x\":6,\"y\":2,\"i\":\"5\"}}",
"weight": 0
}
]
},
{
"id": 0,
"dashboard_id": 0,
"name": "ssh存活探测(配置了ssh探测job才有)",
"weight": 3,
"charts": [
{
"id": 456,
"group_id": 112,
"configs": "{\"name\":\"ssh探测成功个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_success{job=~\\\".*ssh.*\\\"}==1)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}",
"weight": 0
},
{
"id": 457,
"group_id": 112,
"configs": "{\"name\":\"ssh探测失败个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_success{job=~\\\".*ssh.*\\\"}==0)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}",
"weight": 0
},
{
"id": 458,
"group_id": 112,
"configs": "{\"name\":\"ssh探测失败详情\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_success{job=~\\\".*ssh.*\\\"}==0\"],\"layout\":{\"h\":2,\"w\":6,\"x\":12,\"y\":0,\"i\":\"2\"}}",
"weight": 0
},
{
"id": 459,
"group_id": 112,
"configs": "{\"name\":\"ssh探测耗时\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_duration_seconds{job=~\\\".*ssh.*\\\"}\"],\"layout\":{\"h\":2,\"w\":6,\"x\":18,\"y\":0,\"i\":\"3\"}}",
"weight": 0
}
]
},
{
"id": 0,
"dashboard_id": 0,
"name": "icmp探测(配置了icmp探测job才有)",
"weight": 4,
"charts": [
{
"id": 460,
"group_id": 113,
"configs": "{\"name\":\"icmp探测成功个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_success{job=~\\\".*icmp.*\\\"}==1)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}",
"weight": 0
},
{
"id": 461,
"group_id": 113,
"configs": "{\"name\":\"icmp探测失败个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_success{job=~\\\".*icmp.*\\\"}==0)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}",
"weight": 0
},
{
"id": 462,
"group_id": 113,
"configs": "{\"name\":\"icmp探测失败详情\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_success{job=~\\\".*icmp.*\\\"}==0\"],\"layout\":{\"h\":2,\"w\":6,\"x\":12,\"y\":0,\"i\":\"2\"}}",
"weight": 0
},
{
"id": 463,
"group_id": 113,
"configs": "{\"name\":\"icmp探测总耗时\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_duration_seconds{job=~\\\".*icmp.*\\\"}\"],\"layout\":{\"h\":2,\"w\":6,\"x\":18,\"y\":0,\"i\":\"3\"}}",
"weight": 0
}
]
},
{
"id": 0,
"dashboard_id": 0,
"name": "tcp端口探测(配置了tcp探测job才有)",
"weight": 5,
"charts": [
{
"id": 464,
"group_id": 114,
"configs": "{\"name\":\"tcp端口探测成功个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_success{job=~\\\".*tcp.*\\\"}==1)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":0,\"y\":0,\"i\":\"0\"}}",
"weight": 0
},
{
"id": 465,
"group_id": 114,
"configs": "{\"name\":\"tcp端口探测失败个数\",\"mode\":\"promethues\",\"prome_ql\":[\"count(probe_success{job=~\\\".*tcp.*\\\"}==0)\"],\"layout\":{\"h\":2,\"w\":6,\"x\":6,\"y\":0,\"i\":\"1\"}}",
"weight": 0
},
{
"id": 466,
"group_id": 114,
"configs": "{\"name\":\"tcp端口探测失败列表\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_success{job=~\\\".*tcp.*\\\"}==0\"],\"layout\":{\"h\":2,\"w\":6,\"x\":12,\"y\":0,\"i\":\"2\"}}",
"weight": 0
},
{
"id": 467,
"group_id": 114,
"configs": "{\"name\":\"tcp端口探测耗时\",\"mode\":\"promethues\",\"prome_ql\":[\"probe_duration_seconds{job=~\\\".*tcp.*\\\"}\"],\"layout\":{\"h\":2,\"w\":6,\"x\":18,\"y\":0,\"i\":\"3\"}}",
"weight": 0
}
]
}
]
}
]