innov
/
nightingale1
1
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

发布v4版本 (#651) 

* init
This commit is contained in:
qinyening 2021-04-13 11:38:40 +08:00 committed by GitHub
parent bea0532872
commit 59366e4d3a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
760 changed files with 33793 additions and 39323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.gitignore vendored
View File

@ -45,16 +45,6 @@ _test
/n9e-*
/src/modules/agent/agent
/src/modules/ams/ams
/src/modules/index/index
/src/modules/job/job
/src/modules/judge/judge
/src/modules/monapi/monapi
/src/modules/rdb/rdb
/src/modules/transfer/transfer
/src/modules/tsdb/tsdb
tmp/
main
Makefile

14
changelog
View File

@ -126,3 +126,17 @@
升级方法：
- 替换n9e-rdb n9e-prober n9e-monapi n9e-transfer二进制
- 将etc/password-changed-email.tpl放到生产环境的etc目录下
4.0.0
升级内容：
- 服务端模块合并为一个模块
- agentd和server的调用全部走rpc
重新安装：见 https://n9e.didiyun.com/docs/install/binary/
升级方法：
- 使用新的etc替换掉原来的etc
- 使用etc/nginx.conf替换原来的nginx.conf
- n9e-prober替换旧的n9e-prober
- n9e-agentd替换n9e-agent
- n9e-server替换n9e-rdb、n9e-ams、n9e-job、n9e-monapi、n9e-transfer、n9e-judge

49
control
View File

@ -1,7 +1,7 @@
#!/bin/bash
# release version
version=3.8.0
version=4.0.0
CWD=$(cd $(dirname $0)/; pwd)
cd $CWD
@ -14,15 +14,8 @@ usage()
start_all()
{
test -x n9e-rdb && start rdb
test -x n9e-job && start job
test -x n9e-ams && start ams
test -x n9e-monapi && start monapi
test -x n9e-tsdb && start tsdb
test -x n9e-index && start index
test -x n9e-transfer && start transfer
test -x n9e-judge && start judge
test -x n9e-agent && start agent
test -x n9e-server && start server
test -x n9e-agentd && start agentd
test -x n9e-prober && start prober
}
@ -68,16 +61,9 @@ start()
stop_all()
{
test -x n9e-agent && stop agent
test -x n9e-judge && stop judge
test -x n9e-transfer && stop transfer
test -x n9e-index && stop index
test -x n9e-tsdb && stop tsdb
test -x n9e-monapi && stop monapi
test -x n9e-ams && stop ams
test -x n9e-job && stop job
test -x n9e-rdb && stop rdb
test -x n9e-server && stop server
test -x n9e-prober && stop prober
test -x n9e-agentd && stop agentd
}
stop()
@ -160,15 +146,8 @@ build()
mod=$1
if [ "x${mod}" = "x" ]; then
build_one monapi
build_one transfer
build_one index
build_one judge
build_one agent
build_one tsdb
build_one rdb
build_one ams
build_one job
build_one server
build_one agentd
build_one prober
return
fi
@ -182,15 +161,8 @@ build_local()
mod=$1
if [ "x${mod}" = "x" ]; then
build_local_one monapi
build_local_one transfer
build_local_one index
build_local_one judge
build_local_one agent
build_local_one tsdb
build_local_one rdb
build_local_one ams
build_local_one job
build_local_one server
build_local_one agentd
build_local_one prober
return
fi
@ -242,8 +214,9 @@ exec()
fi
done
else
echo $1
echo "todo $1 at "$(date "+%Y-%m-%d %H:%M:%S")
$1
echo "done $1 at "$(date "+%Y-%m-%d %H:%M:%S")
fi
}

45
etc/address.yml
View File

@ -1,51 +1,12 @@
---
rdb:
server:
http: 0.0.0.0:8000
addresses:
- 127.0.0.1
ams:
http: 0.0.0.0:8002
addresses:
- 127.0.0.1
job:
http: 0.0.0.0:8004
rpc: 0.0.0.0:8005
addresses:
- 127.0.0.1
monapi:
http: 0.0.0.0:8006
addresses:
- 127.0.0.1
transfer:
http: 0.0.0.0:8008
rpc: 0.0.0.0:8009
addresses:
- 127.0.0.1
tsdb:
http: 0.0.0.0:8010
rpc: 0.0.0.0:8011
index:
http: 0.0.0.0:8012
rpc: 0.0.0.0:8013
addresses:
- 127.0.0.1
judge:
http: 0.0.0.0:8014
rpc: 0.0.0.0:8015
rpc: 0.0.0.0:8001
addresses:
- 127.0.0.1
prober:
http: 0.0.0.0:8023
addresses:
- 127.0.0.1
agent:
agentd:
http: 0.0.0.0:2080

3
etc/agent.yml → etc/agentd.yml
View File

@ -1,5 +1,5 @@
logger:
dir: logs/agent
dir: logs/agentd
level: INFO
keepHours: 24
@ -75,5 +75,6 @@ sys:
- cpu.core.softirq
- cpu.core.iowait
- cpu.core.steal
#ntpServers:
# - ntp1.aliyun.com

12
etc/ams.yml
View File

@ -1,12 +0,0 @@
logger:
dir: logs/ams
level: INFO
keepHours: 24
http:
mode: release
cookieDomain: ""
cookieName: ecmc-sid
tokens:
- ams-builtin-token

119
etc/dict.json
View File

@ -1,118 +1 @@
{
"zh": {
"stra not found": "聚合策略为找到",
"same stra name %s in node": "同节点下策略名称 %s 已存在",
"collect type not support": "采集类型不合法",
"[%s] is blank": "参数[%s]值不能为空",
"cannot convert %s to int64": "%s 无法转为 int64 类型",
"cannot convert %s to int": "%s 无法转为 int 类型",
"arg[%s] not found": "参数[%s]没找到",
"cannot retrieve node[%d]: %v": "获取不到节点[%d],原因:%v",
"no such node[%d]": "节点[%d]不存在",
"no such task[id:%d]": "任务[%d]不存在",
"no such task tpl[id:%d]": "任务模板[%d]不存在",
"cannot retrieve screen[%d]: %v": "获取不到大盘[%d],原因:%v",
"no such screen[%d]": "大盘[%d]不存在",
"cannot retrieve subclass[%d]: %v": "获取不到大盘分组[%d],原因:%v",
"no such subclass[%d]": "大盘分组[%d]不存在",
"cannot retrieve chart[%d]: %v": "获取不到大盘图表[%d],原因:%v",
"no such chart[%d]": "大盘图表[%d]不存在",
"cannot retrieve eventCur[%d]: %v": "获取不到未恢复告警事件[%d],原因:%v",
"no such eventCur[%d]": "未恢复告警事件[%d]不存在",
"cannot retrieve event[%d]: %v": "获取不到告警事件[%d],原因:%v",
"no such event[%d]": "告警事件[%d]不存在",
"cannot retrieve user[%d]: %v": "获取不到用户[%d],原因:%v",
"no such user[%d]": "用户[%d]不存在",
"no such user: %s": "用户[%s]不存在",
"cannot retrieve team[%d]: %v": "获取不到团队[%d],原因:%v",
"no such team[%d]": "团队[%d]不存在",
"cannot retrieve role[%d]: %v": "获取不到角色[%d],原因:%v",
"no such role[%d]": "角色[%d]不存在",
"no such NodeCate[id:%d]": "节点类型[%d]没找到",
"no such field": "扩展字段为找到",
"field_type cannot modify": "字段类型不能被修改",
"arg[endpoints] empty": "参数不能[endpoints]为空",
"arg[cur_nid_paths] empty": "参数不能[cur_nid_paths]为空",
"arg[tags] empty": "参数不能[tags]为空",
"arg[hosts] empty": "参数不能[hosts]为空",
"arg[btime,etime] empty": "参数[btime,etime]不合规范",
"arg[name] empty": "参数[name]不合规范",
"arg[name] is blank": "参数[名称]不能为空",
"arg[ids] is empty": "参数[ids]不能为空",
"%s invalid": "%s 不符合规范",
"%s too long > 64": "%s 超过64长度限制",
"arg[%s] too long > %d": "参数 %s 长度不能超过 %d",
"cate is blank": "节点分类不能为空",
"uuid is blank": "uuid不能为空",
"ident is blank": "唯一标识不能为空",
"tenant is blank": "租户不能为空",
"ids is blank": "ids不能为空",
"items empty": "提交内容不能为空",
"url param[%s] is blank": "url参数[%s]不能为空",
"query param[%s] is necessary": "query参数[%s]不能为空",
"ident legal characters: [a-z0-9_-]": "唯一标识英文只能字母开头，包括数字、中划线、下划线",
"ident length should be less than 32": "唯一标识长度需小于32",
"cannot modify tenant's node-category": "租户分类不允许修改",
"cannot modify node-category to tenant": "节点分类不允许修改为租户",
"node is managed by other system": "租户正在被系统系统使用",
"resources not found by %s": "通过 %s 没有找到资源",
"cannot delete root user": "root用户不能删除",
"user not found": "用户未找到",
"Unable to get captcha": "无法获得验证码",
"Invalid captcha answer": "错误的验证码",
"Username %s is invalid": "用户名 %s 不符合规范",
"Username %s too long > 64": "用户名 %s 太长(64)",
"Unable to get login arguments": "无法获得登陆参数",
"Deny Access from %s with whitelist control": "来自 %s 的访问被白名单规则拒绝",
"Invalid login type %s": "不支持的登陆类型 %s",
"Unable to get type, sms-code | email-code": "无法获得验证码类型",
"Unable to get code arg": "无法获得验证码类型",
"sms/email sender is disabled": "无法发送 短信/邮件 验证码",
"Invalid code type %s": "不支持的验证码类型 %s",
"Cannot find the user by %s": "无法用 %s 找到相关用户",
"Unable to get password": "无法获取密码",
"Invalid code": "不符合规范的验证码",
"The code is incorrect": "无效的验证码",
"The code has expired": "失效的验证码",
"Invalid arguments %s": "不合法的参数 %s",
"Login fail, check your username and password": "登陆失败，请检查用户名/密码",
"User dose not exist": "用户不存在",
"Username %s already exists": "用户名 %s 已存在",
"Upper char": "大写字母",
"Lower char": "小写字母",
"Number": "数字",
"Special char": "特殊字符",
"Must include %s": "必须包含 %s",
"Invalid Password, %s": "密码不符合规范, %s",
"character: %s not supported": "不支持的字符 %s",
"Incorrect login/password %s times, you still have %s chances": "登陆失败%d次，你还有%d次机会",
"The limited sessions %d": "会话数量限制，最多%d个会话",
"Password has been expired": "密码已过期，请重置密码",
"User is inactive": "用户已禁用",
"User is locked": "用户已锁定",
"User is frozen": "用户已休眠",
"User is writen off": "用户已注销",
"Minimum password length %d": "密码最小长度 %d",
"Password too short (min:%d) %s": "密码太短 (最小 %d) %s",
"%s format error":"%s 所填内容不符合规范",
"%s %s format error":"%s %s 所填内容不符合规范",
"username too long (max:%d)": "用户名太长 (最长:%d)",
"dispname too long (max:%d)": "昵称太长 (最长:%d)",
"email %s or phone %s is exists": "邮箱 %s 或者 手机号 %s 已存在",
"Password is not set": "密码未设置",
"Incorrect old password": "密码错误",
"The password is the same as the old password": "密码与历史密码重复",
"phone": "手机号",
"email": "邮箱",
"username": "用户名",
"dispname": "昵称",
"Temporary user has expired": "临时账户,已过有效期",
"Invalid user status %d": "异常的用户状态 %d",
"Password expired, please change the password in time": "密码过期，请及时修改密码",
"First Login, please change the password in time": "初始登陆，请及时修改密码",
"EOF": ""
}
}
{}

4
etc/identity.yml
View File

@ -1,9 +1,9 @@
# 用来做心跳，给服务端上报本机ip
# for heartbeat, connected by other modules
ip:
specify: ""
shell: ifconfig `route|grep '^default'|awk '{print $NF}'`|grep inet|awk '{print $2}'|head -n 1|awk -F':' '{print $NF}'
# MON、JOB, judge, prober 的客户端拿来做本机标识
# as identity. equals to endpoint. used by agentd, prober, server
ident:
specify: ""
shell: ifconfig `route|grep '^default'|awk '{print $NF}'`|grep inet|awk '{print $2}'|head -n 1|awk -F':' '{print $NF}'

4
etc/index.yml
View File

@ -1,4 +0,0 @@
logger:
dir: logs/index
level: INFO
keepHours: 24

15
etc/job.yml
View File

@ -1,15 +0,0 @@
logger:
dir: logs/job
level: INFO
keepHours: 24
http:
mode: release
cookieDomain: ""
cookieName: ecmc-sid
output:
# database | remote
comeFrom: database
remotePort: 2080

410
etc/json/stra.json
View File

@ -1,410 +0,0 @@
[
{
"name": "内存利用率大于75%",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 2,
"exprs": [
{
"eopt": ">",
"func": "all",
"metric": "mem.bytes.used.percent",
"params": [],
"threshold": 75
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "机器loadavg大于16",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 2,
"exprs": [
{
"eopt": ">",
"func": "all",
"metric": "cpu.loadavg.1",
"params": [],
"threshold": 16
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "某磁盘无法正常读写",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 1,
"exprs": [
{
"eopt": ">",
"func": "all",
"metric": "disk.rw.error",
"params": [],
"threshold": 0
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "监控agent失联",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 1,
"exprs": [
{
"eopt": "=",
"func": "nodata",
"metric": "proc.agent.alive",
"params": [],
"threshold": 0
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "磁盘利用率达到85%",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 3,
"exprs": [
{
"eopt": ">",
"func": "all",
"metric": "disk.bytes.used.percent",
"params": [],
"threshold": 85
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "磁盘利用率达到88%",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 2,
"exprs": [
{
"eopt": ">",
"func": "all",
"metric": "disk.bytes.used.percent",
"params": [],
"threshold": 88
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "磁盘利用率达到92%",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 1,
"exprs": [
{
"eopt": ">",
"func": "all",
"metric": "disk.bytes.used.percent",
"params": [],
"threshold": 92
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "端口挂了",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 2,
"exprs": [
{
"eopt": "!=",
"func": "all",
"metric": "proc.port.listen",
"params": [],
"threshold": 1
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "网卡入方向丢包",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 2,
"exprs": [
{
"eopt": ">",
"func": "all",
"metric": "net.in.dropped",
"params": [],
"threshold": 3
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "网卡出方向丢包",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 2,
"exprs": [
{
"eopt": ">",
"func": "all",
"metric": "net.out.dropped",
"params": [],
"threshold": 3
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "进程总数超过3000",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 1,
"exprs": [
{
"eopt": ">",
"func": "all",
"metric": "sys.ps.process.total",
"params": [],
"threshold": 3000
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
},
{
"name": "进程挂了",
"category": 1,
"alert_dur": 60,
"recovery_dur": 0,
"recovery_notify": 1,
"enable_stime": "00:00",
"enable_etime": "23:59",
"priority": 2,
"exprs": [
{
"eopt": "<",
"func": "all",
"metric": "proc.num",
"params": [],
"threshold": 1
}
],
"tags": [],
"enable_days_of_week": [
0,
1,
2,
3,
4,
5,
6
],
"converge": [
36000,
1
],
"endpoints": null
}
]

18
etc/judge.yml
View File

@ -1,18 +0,0 @@
query:
connTimeout: 1000
callTimeout: 2000
indexCallTimeout: 2000
redis:
addrs:
- 127.0.0.1:6379
pass: ""
# timeout:
# conn: 500
# read: 3000
# write: 3000
logger:
dir: logs/judge
level: INFO
keepHours: 24

47
etc/monapi.yml
View File

@ -1,47 +0,0 @@
---
tokens:
- monapi-internal-third-module-pass-fjsdi
logger:
dir: logs/monapi
level: INFO
keepHours: 24
alarmEnabled: true
region:
- default
# clean history event
cleaner:
# retention days
days: 100
# number of events deleted per time
batch: 100
# read alert from redis
redis:
addr: 127.0.0.1:6379
pass: ""
# timeout:
# conn: 500
# read: 3000
# write: 3000
i18n:
lang: zh
notify:
p1: ["voice", "sms", "mail", "im"]
p2: ["sms", "mail", "im"]
p3: ["mail", "im"]
# addresses accessible using browser
link:
stra: http://n9e.com/mon/strategy/%v
event: http://n9e.com/mon/history/his/%v
claim: http://n9e.com/mon/history/cur/%v
http:
mode: release
cookieDomain: ""
cookieName: ecmc-sid

12
etc/nginx.conf
View File

@ -67,33 +67,33 @@ http {
}
upstream n9e.ams {
server 127.0.0.1:8002;
server 127.0.0.1:8000;
keepalive 60;
}
upstream n9e.job {
server 127.0.0.1:8004;
server 127.0.0.1:8000;
keepalive 60;
}
upstream n9e.monapi {
server 127.0.0.1:8006;
server 127.0.0.1:8000;
keepalive 60;
}
upstream n9e.transfer {
server 127.0.0.1:8008;
server 127.0.0.1:8000;
keepalive 60;
}
upstream n9e.index {
server 127.0.0.1:8012;
server 127.0.0.1:8000;
keepalive 60;
}
server {
listen 80 default_server;
server_name n9e.example.com;
server_name localhost;
root /home/n9e/pub;
# Load configuration files for the default server block.

110
etc/rdb.yml
View File

@ -1,110 +0,0 @@
logger:
dir: logs/rdb
level: INFO
keepHours: 24
http:
mode: release
session:
cookieName: ecmc-sid
domain: ""
httpOnly: true
gcInterval: 60
cookieLifetime: 86400 # 单位秒，0: 与浏览器相同
i18n:
lang: zh
sso:
enable: false
ssoAddr: "http://{sso-host}"
redirectURL: "http://{rdb-host}/auth-callback"
clientId: ""
clientSecret: ""
apiKey: ""
attributes:
dispname: "display_name"
email: "email"
phone: "phone"
im: ""
coverAttributes: false
stateExpiresIn: 300
auth:
captcha: false
extraMode:
enable: false # enable whiteList, login retry lock, userControl, ...
whiteList: false
frozenDays: 90 # frozen time (day)
writenOffDays: 365 # writenOff time (day)
tokens:
- rdb-builtin-token
# for ldap authorization
ldap:
host: "ldap.example.org"
port: 389
baseDn: "dc=example,dc=org"
# AD: manange@example.org
bindUser: "cn=manager,dc=example,dc=org"
bindPass: "*******"
# openldap: (&(uid=%s))
# AD: (&(sAMAccountName=%s))
authFilter: "(&(uid=%s))"
attributes:
dispname: "cn"
email: "mail"
phone: "mobile"
im: ""
coverAttributes: false
autoRegist: true
tls: false
startTLS: false
# as queue for sender
redis:
enable: true
addr: 127.0.0.1:6379
pass: ""
idle: 5
timeout:
conn: 500
read: 3000
write: 3000
rabbitmq:
enable: false
addr: amqp://root:1234@127.0.0.1:5672/
queue: test
sender:
mail:
# three choice: smtp|shell|api
way: smtp
worker: 10
api: http://127.0.0.1:2008/mail
sms:
# two choice: shell|api
way: api
worker: 10
api: http://127.0.0.1:2008/sms
voice:
# two choice: shell|api
way: shell
worker: 10
api: http://127.0.0.1:2008/voice
im:
# five choice: shell|api|wechat|wechat_robot|dingtalk_robot
way: shell
worker: 10
api: http://127.0.0.1:2008/im
wechat:
corp_id: "xxxxxxxxxxxxx"
agent_id: 1000000
secret: "xxxxxxxxxxxxxxxxx"
#webhook:
# - addr: http://localhost:2008/api/event
# token: "1234"

88
etc/screen/n9e_mudules
View File

@ -1,88 +0,0 @@
[
{
"name": "N9E模块监控",
"node_path": "",
"tags": [
{
"name": "transfer",
"weight": 0,
"charts": [
{
"configs": "{\"title\":\"transfer接收点数\",\"type\":\"chart\",\"now\":\"1614909881267\",\"start\":\"1614906281267\",\"end\":\"1614909881267\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":7,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.transfer.points.in\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1,\"aggrFunc\":\"max\"}],\"chartTypeOptions\":{\"chartType\":\"singleValue\",\"targetValue\":\"max\",\"subType\":\"normal\",\"valueMap\":\"range\",\"mapConf\":[{\"color\":\"#06c658\",\"from\":0,\"to\":50000},{\"from\":50000,\"to\":800000,\"color\":\"#da8e58\"},{\"from\":80000,\"to\":100000,\"color\":\"#f48b71\"},{\"from\":10000,\"color\":\"#fa0505\"}]},\"id\":25}",
"weight": 0
},
{
"configs": "{\"title\":\"transfer发送tsdb点数\",\"type\":\"chart\",\"now\":\"1614909881267\",\"start\":\"1614906281267\",\"end\":\"1614909881267\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":7,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.transfer.points.out.tsdb\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1,\"aggrFunc\":\"max\"}],\"chartTypeOptions\":{\"chartType\":\"singleValue\",\"targetValue\":\"max\",\"subType\":\"normal\",\"valueMap\":\"range\",\"mapConf\":[{\"color\":\"#06c658\",\"from\":0,\"to\":50000},{\"from\":50000,\"to\":800000,\"color\":\"#da8e58\"},{\"from\":80000,\"to\":100000,\"color\":\"#f48b71\"},{\"from\":10000,\"color\":\"#fa0505\"}]},\"id\":26}",
"weight": 1
},
{
"configs": "{\"title\":\"transfer向judge发送的点数\",\"type\":\"chart\",\"now\":\"1614909881267\",\"start\":\"1614906281267\",\"end\":\"1614909881267\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":7,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.transfer.points.out.judge\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1,\"aggrFunc\":\"max\"}],\"chartTypeOptions\":{\"chartType\":\"singleValue\",\"targetValue\":\"max\",\"subType\":\"normal\",\"valueMap\":\"range\",\"mapConf\":[{\"color\":\"#06c658\",\"from\":0,\"to\":50000},{\"from\":50000,\"to\":800000,\"color\":\"#da8e58\"},{\"from\":80000,\"to\":100000,\"color\":\"#f48b71\"},{\"from\":10000,\"color\":\"#fa0505\"}]},\"id\":28}",
"weight": 2
},
{
"configs": "{\"title\":\"\",\"type\":\"chart\",\"now\":\"1614667049309\",\"start\":\"1614663449309\",\"end\":\"1614667049309\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":7,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.transfer.get.index.err\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1}]}",
"weight": 3
},
{
"configs": "{\"title\":\"\",\"type\":\"chart\",\"now\":\"1614667049309\",\"start\":\"1614663449309\",\"end\":\"1614667049309\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":7,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.transfer.judge.get.err\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1}]}",
"weight": 4
},
{
"configs": "{\"title\":\"transfer向judge发送的点数\",\"type\":\"chart\",\"now\":\"1614909881267\",\"start\":\"1614906281267\",\"end\":\"1614909881267\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":7,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.transfer.stra.count\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1,\"aggrFunc\":\"max\"}],\"chartTypeOptions\":{\"chartType\":\"singleValue\",\"targetValue\":\"max\",\"subType\":\"normal\",\"valueMap\":\"range\",\"mapConf\":[{\"color\":\"#06c658\",\"from\":0,\"to\":5000},{\"from\":5000,\"to\":80000,\"color\":\"#da8e58\"},{\"from\":8000,\"to\":10000,\"color\":\"#f48b71\"},{\"from\":10000,\"color\":\"#fa0505\"}]},\"id\":54}",
"weight": 5
}
]
},
{
"name": "tsdb",
"weight": 1,
"charts": [
{
"configs": "{\"title\":\"\",\"type\":\"chart\",\"now\":\"1614909881267\",\"start\":\"1614906281267\",\"end\":\"1614909881267\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":7,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.tsdb.points.in\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1,\"aggrFunc\":\"max\"}],\"chartTypeOptions\":{\"chartType\":\"singleValue\",\"targetValue\":\"max\",\"subType\":\"normal\",\"valueMap\":\"range\",\"mapConf\":[{\"from\":0,\"to\":50000,\"color\":\"#06c663\"},{\"from\":50000,\"to\":80000,\"color\":\"#da8e0b\"},{\"from\":80000,\"to\":100000,\"color\":\"#f48b71\"},{\"from\":100000,\"color\":\"#fa0505\"}]},\"id\":33}",
"weight": 0
},
{
"configs": "{\"title\":\"\",\"type\":\"chart\",\"now\":\"1614909881267\",\"start\":\"1614906281267\",\"end\":\"1614909881267\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":7,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.tsdb.index.out\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1,\"aggrFunc\":\"avg\"}],\"chartTypeOptions\":{\"chartType\":\"singleValue\",\"targetValue\":\"avg\",\"subType\":\"normal\",\"valueMap\":\"range\",\"mapConf\":[{\"from\":0,\"to\":50000,\"color\":\"#06c663\"},{\"from\":50000,\"to\":80000,\"color\":\"#da8e0b\"},{\"from\":80000,\"to\":100000,\"color\":\"#f48b71\"},{\"from\":100000,\"color\":\"#fa0505\"}]},\"id\":34}",
"weight": 1
},
{
"configs": "{\"title\":\"\",\"type\":\"chart\",\"now\":\"1614909881267\",\"start\":\"1614906281267\",\"end\":\"1614909881267\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":7,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.tsdb.query.hit.cache\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1,\"aggrFunc\":\"avg\"}],\"chartTypeOptions\":{\"chartType\":\"singleValue\",\"targetValue\":\"avg\",\"subType\":\"normal\",\"valueMap\":\"range\",\"mapConf\":[{\"from\":0,\"to\":50000,\"color\":\"#06c663\"},{\"from\":50000,\"to\":80000,\"color\":\"#da8e0b\"},{\"from\":80000,\"to\":100000,\"color\":\"#f48b71\"},{\"from\":100000,\"color\":\"#fa0505\"}]},\"id\":35}",
"weight": 2
}
]
},
{
"name": "Judge",
"weight": 2,
"charts": [
{
"configs": "{\"title\":\"judge接收点数\",\"type\":\"chart\",\"now\":\"1614909881045\",\"start\":\"1614906281045\",\"end\":\"1614909881045\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":5,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.judge.push.in\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1}]}",
"weight": 0
},
{
"configs": "{\"title\":\"正在执行的judge任务数\",\"type\":\"chart\",\"now\":\"1614909881045\",\"start\":\"1614906281045\",\"end\":\"1614909881045\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":5,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.judge.running\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1}]}",
"weight": 1
},
{
"configs": "{\"title\":\"获取的策略数\",\"type\":\"chart\",\"now\":\"1614909881045\",\"start\":\"1614906281045\",\"end\":\"1614909881045\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":5,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.judge.stra.count\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1,\"aggrFunc\":\"max\"}],\"chartTypeOptions\":{\"chartType\":\"singleValue\",\"targetValue\":\"max\",\"subType\":\"normal\",\"valueMap\":\"range\",\"mapConf\":[{\"from\":0,\"to\":5000,\"color\":\"#06c663\"},{\"from\":5000,\"to\":8000,\"color\":\"#f88e0b\"},{\"from\":8000,\"to\":10000,\"color\":\"#f48b71\"},{\"from\":10000,\"color\":\"#fa0505\"}]},\"id\":60}",
"weight": 2
}
]
},
{
"name": "index",
"weight": 3,
"charts": [
{
"configs": "{\"title\":\"fullmatch接口查索引未命中次数\",\"type\":\"chart\",\"now\":\"1614909881045\",\"start\":\"1614906281045\",\"end\":\"1614909881045\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":5,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.index.query.counter.miss\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1}]}",
"weight": 0
},
{
"configs": "{\"title\":\"xclude接口查索引命中次数\",\"type\":\"chart\",\"now\":\"1614909881045\",\"start\":\"1614906281045\",\"end\":\"1614909881045\",\"comparisonOptions\":[{\"label\":\"1小时\",\"labelEn\":\"1 hour\",\"value\":\"3600000\"},{\"label\":\"2小时\",\"labelEn\":\"2 hours\",\"value\":\"7200000\"},{\"label\":\"1天\",\"labelEn\":\"1 day\",\"value\":\"86400000\"},{\"label\":\"2天\",\"labelEn\":\"2 days\",\"value\":\"172800000\"},{\"label\":\"7天\",\"labelEn\":\"7 days\",\"value\":\"604800000\"}],\"legend\":false,\"shared\":true,\"linkVisible\":true,\"metrics\":[{\"selectedNid\":5,\"endpointsKey\":\"endpoints\",\"selectedEndpoint\":[\"=all\"],\"selectedMetric\":\"n9e.index.xclude.qp10s\",\"selectedTagkv\":[{\"tagk\":\"endpoint\",\"tagv\":[\"=all\"]}],\"counterListCount\":1}]}",
"weight": 1
}
]
}
]
}
]

172
etc/server.yml Normal file
View File

@ -0,0 +1,172 @@
logger:
dir: logs/server
level: INFO
# rotate by time
keepHours: 24
## rotate by size
#rotatenum: 3
#rotatesize: 256 # unit: MB
http:
mode: release
showLog: true
session:
cookieName: ecmc-sid
cookieDomain: ""
httpOnly: true
gcInterval: 60
cookieLifetime: 86400 # unit: second, 0: cookie life same to browser
tokens:
- rdb-builtin-token
redis:
# as queue
local:
enable: true
addr: 127.0.0.1:6379
pass: ""
idle: 5
timeout:
conn: 500
read: 3000
write: 3000
rdb:
# for ldap authorization
ldap:
host: "ldap.example.org"
port: 389
baseDn: "dc=example,dc=org"
# AD: manange@example.org
bindUser: "cn=manager,dc=example,dc=org"
bindPass: "*******"
# openldap: (&(uid=%s))
# AD: (&(sAMAccountName=%s))
authFilter: "(&(uid=%s))"
attributes:
dispname: "cn"
email: "mail"
phone: "mobile"
im: ""
coverAttributes: false
autoRegist: true
tls: false
startTLS: false
## for enterprise edition
#sso:
# enable: false
# ssoAddr: "http://{sso-host}"
# redirectURL: "http://{rdb-host}/auth-callback"
# clientId: ""
# clientSecret: ""
# apiKey: ""
# attributes:
# dispname: "display_name"
# email: "email"
# phone: "phone"
# im: ""
# coverAttributes: false
# stateExpiresIn: 300
#auth:
# captcha: false
# extraMode:
# enable: false # enable whiteList, login retry lock, userControl, ...
# whiteList: false
# frozenDays: 90 # frozen time (day)
# writenOffDays: 365 # writenOff time (day)
#i18n:
# lang: zh
job:
enable: true
## database | remote
#outputComeFrom: database
#remoteAgtdPort: 2080
transfer:
enable: true
backend:
datasource: "m3db"
m3db:
enabled: true
maxSeriesPoints: 720 # default 720
name: "m3db"
namespace: "default"
seriesLimit: 0
docsLimit: 0
daysLimit: 7 # max query time
# https://m3db.github.io/m3/m3db/architecture/consistencylevels/
writeConsistencyLevel: "majority" # one|majority|all
readConsistencyLevel: "unstrict_majority" # one|unstrict_majority|majority|all
config:
service:
# KV environment, zone, and service from which to write/read KV data (placement
# and configuration). Leave these as the default values unless you know what
# you're doing.
env: default_env
zone: embedded
service: m3db
etcdClusters:
- zone: embedded
endpoints:
- 127.0.0.1:2379
#tls:
# caCrtPath: /etc/etcd/certs/ca.pem
# crtPath: /etc/etcd/certs/etcd-client.pem
# keyPath: /etc/etcd/certs/etcd-client-key.pem
#tsdb:
# enabled: false
# name: "tsdb"
# cluster:
# tsdb01: 127.0.0.1:8011
#influxdb:
# enabled: false
# username: "influx"
# password: "admin123"
# precision: "s"
# database: "n9e"
# address: "http://127.0.0.1:8086"
#opentsdb:
# enabled: false
# address: "127.0.0.1:4242"
#kafka:
# enabled: false
# brokersPeers: "192.168.1.1:9092,192.168.1.2:9092"
# topic: "n9e"
monapi:
indexMod: server
alarmEnabled: true
region:
- default
# clean history event
cleaner:
# retention days
days: 100
# number of events deleted per time
batch: 100
notify:
p1: ["voice", "sms", "mail", "im"]
p2: ["sms", "mail", "im"]
p3: ["mail", "im"]
# addresses accessible using browser
link:
stra: http://n9e.com/mon/strategy/%v
event: http://n9e.com/mon/history/his/%v
claim: http://n9e.com/mon/history/cur/%v
judge:
query:
connTimeout: 1000
callTimeout: 2000
maxConn: 2000
maxIdle: 100
connTimeout: 1000
callTimeout: 2000
indexCallTimeout: 2000
indexMod: server

2
etc/service/agent.service → etc/service/agentd.service
View File

@ -10,7 +10,7 @@ Group=root
Type=simple
Environment="GIN_MODE=release"
ExecStart=/home/n9e/n9e-agent
ExecStart=/home/n9e/n9e-agentd
WorkingDirectory=/home/n9e
Restart=always

21
etc/service/job.service
View File

@ -1,21 +0,0 @@
[Unit]
Description=n9e job
After=network-online.target
Wants=network-online.target
[Service]
# modify when deploy in prod env
User=root
Group=root
Type=simple
Environment="GIN_MODE=release"
ExecStart=/home/n9e/n9e-job
WorkingDirectory=/home/n9e
Restart=always
RestartSec=1
StartLimitInterval=0
[Install]
WantedBy=multi-user.target

21
etc/service/judge.service
View File

@ -1,21 +0,0 @@
[Unit]
Description=n9e judge
After=network-online.target
Wants=network-online.target
[Service]
# modify when deploy in prod env
User=root
Group=root
Type=simple
Environment="GIN_MODE=release"
ExecStart=/home/n9e/n9e-judge
WorkingDirectory=/home/n9e
Restart=always
RestartSec=1
StartLimitInterval=0
[Install]
WantedBy=multi-user.target

21
etc/service/monapi.service
View File

@ -1,21 +0,0 @@
[Unit]
Description=n9e monapi
After=network-online.target
Wants=network-online.target
[Service]
# modify when deploy in prod env
User=root
Group=root
Type=simple
Environment="GIN_MODE=release"
ExecStart=/home/n9e/n9e-monapi
WorkingDirectory=/home/n9e
Restart=always
RestartSec=1
StartLimitInterval=0
[Install]
WantedBy=multi-user.target

4
etc/service/index.service → etc/service/prober.service
View File

@ -1,5 +1,5 @@
[Unit]
Description=n9e index
Description=n9e prober
After=network-online.target
Wants=network-online.target
@ -10,7 +10,7 @@ Group=root
Type=simple
Environment="GIN_MODE=release"
ExecStart=/home/n9e/n9e-index
ExecStart=/home/n9e/n9e-prober
WorkingDirectory=/home/n9e
Restart=always

21
etc/service/rdb.service
View File

@ -1,21 +0,0 @@
[Unit]
Description=n9e rdb
After=network-online.target
Wants=network-online.target
[Service]
# modify when deploy in prod env
User=root
Group=root
Type=simple
Environment="GIN_MODE=release"
ExecStart=/home/n9e/n9e-rdb
WorkingDirectory=/home/n9e
Restart=always
RestartSec=1
StartLimitInterval=0
[Install]
WantedBy=multi-user.target

4
etc/service/ams.service → etc/service/server.service
View File

@ -1,5 +1,5 @@
[Unit]
Description=n9e ams
Description=n9e server
After=network-online.target
Wants=network-online.target
@ -10,7 +10,7 @@ Group=root
Type=simple
Environment="GIN_MODE=release"
ExecStart=/home/n9e/n9e-ams
ExecStart=/home/n9e/n9e-server
WorkingDirectory=/home/n9e
Restart=always

21
etc/service/transfer.service
View File

@ -1,21 +0,0 @@
[Unit]
Description=n9e transfer
After=network-online.target
Wants=network-online.target
[Service]
# modify when deploy in prod env
User=root
Group=root
Type=simple
Environment="GIN_MODE=release"
ExecStart=/home/n9e/n9e-transfer
WorkingDirectory=/home/n9e
Restart=always
RestartSec=1
StartLimitInterval=0
[Install]
WantedBy=multi-user.target

21
etc/service/tsdb.service
View File

@ -1,21 +0,0 @@
[Unit]
Description=n9e tsdb
After=network-online.target
Wants=network-online.target
[Service]
# modify when deploy in prod env
User=root
Group=root
Type=simple
Environment="GIN_MODE=release"
ExecStart=/home/n9e/n9e-tsdb
WorkingDirectory=/home/n9e
Restart=always
RestartSec=1
StartLimitInterval=0
[Install]
WantedBy=multi-user.target

52
etc/transfer.yml
View File

@ -1,52 +0,0 @@
backend:
datasource: "tsdb"
m3db:
enabled: false
maxSeriesPoints: 720 # default 720
name: "m3db"
namespace: "default"
seriesLimit: 0
docsLimit: 0
daysLimit: 7 # max query time
# https://m3db.github.io/m3/m3db/architecture/consistencylevels/
writeConsistencyLevel: "majority" # one|majority|all
readConsistencyLevel: "unstrict_majority" # one|unstrict_majority|majority|all
config:
service:
# KV environment, zone, and service from which to write/read KV data (placement
# and configuration). Leave these as the default values unless you know what
# you're doing.
env: default_env
zone: embedded
service: m3db
etcdClusters:
- zone: embedded
endpoints:
- 127.0.0.1:2379
tls:
caCrtPath: /etc/etcd/certs/ca.pem
crtPath: /etc/etcd/certs/etcd-client.pem
keyPath: /etc/etcd/certs/etcd-client-key.pem
tsdb:
enabled: true
name: "tsdb"
cluster:
tsdb01: 127.0.0.1:8011
influxdb:
enabled: false
username: "influx"
password: "admin123"
precision: "s"
database: "n9e"
address: "http://127.0.0.1:8086"
opentsdb:
enabled: false
address: "127.0.0.1:4242"
kafka:
enabled: false
brokersPeers: "192.168.1.1:9092,192.168.1.2:9092"
topic: "n9e"
logger:
dir: logs/transfer
level: INFO
keepHours: 24

8
etc/tsdb.yml
View File

@ -1,8 +0,0 @@
rrd:
storage: data/8011
cache:
keepMinutes: 120
logger:
dir: logs/tsdb
level: WARNING
keepHours: 2

14
go.mod
View File

@ -1,20 +1,21 @@
module github.com/didi/nightingale
module github.com/didi/nightingale/v4
go 1.12
require (
github.com/Shopify/sarama v1.27.2
github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c // indirect
github.com/cespare/xxhash v1.1.0
github.com/codegangsta/negroni v1.0.0
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/dgryski/go-tsz v0.0.0-20180227144327-03b7d791f4fe
github.com/freedomkk-qfeng/go-fastping v0.0.0-20160109021039-d7bb493dee3e // indirect
github.com/gaochao1/gosnmp v0.0.0-20150630013918-783a67a067fd // indirect
github.com/gaochao1/sw v4.0.0+incompatible
github.com/garyburd/redigo v1.6.2
github.com/gin-contrib/pprof v1.3.0
github.com/gin-gonic/gin v1.6.3
github.com/go-ping/ping v0.0.0-20201115131931-3300c582a663
github.com/go-sql-driver/mysql v1.5.0
github.com/google/uuid v1.1.2
github.com/gorilla/mux v1.7.3
github.com/hashicorp/golang-lru v0.5.4
github.com/hpcloud/tail v1.0.0
github.com/influxdata/influxdb v1.8.0
@ -23,17 +24,18 @@ require (
github.com/mattn/go-isatty v0.0.12
github.com/mattn/go-sqlite3 v1.14.0 // indirect
github.com/mojocn/base64Captcha v1.3.1
github.com/open-falcon/rrdlite v0.0.0-20200214140804-bf5829f786ad
github.com/pquerna/cachecontrol v0.0.0-20200819021114-67c6ae64274f // indirect
github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62 // indirect
github.com/shirou/gopsutil v3.20.11+incompatible // indirect
github.com/spaolacci/murmur3 v1.1.0
github.com/sparrc/go-ping v0.0.0-20190613174326-4e5b6552494c
github.com/spf13/viper v1.7.1
github.com/streadway/amqp v1.0.0
github.com/stretchr/testify v1.6.1
github.com/toolkits/file v0.0.0-20160325033739-a5b3c5147e07 // indirect
github.com/toolkits/pkg v1.1.3
github.com/toolkits/sys v0.0.0-20170615103026-1f33b217ffaf // indirect
github.com/ugorji/go/codec v1.1.7
github.com/unrolled/render v1.0.3
go.uber.org/automaxprocs v1.3.0 // indirect
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/text v0.3.3

24
go.sum
View File

@ -109,6 +109,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2c
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5 h1:P5U+E4x5OkVEKQDklVPmzs71WM56RTTRqV4OrDC//Y4=
github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5/go.mod h1:976q2ETgjT2snVCf2ZaBnyBbVoPERGjUz+0sofzEfro=
github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c h1:k/7/05/5kPRX7HaKyVYlsGVX6XkFTyYLqkqHzceUVlU=
github.com/alouca/gologger v0.0.0-20120904114645-7d4b7291de9c/go.mod h1:SI1d/2/wpSTDjHgdS9ZLy6hqvsdhzVYAc8RLztweMpA=
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
@ -184,8 +186,6 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY=
github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@ -224,8 +224,6 @@ github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/
github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dgryski/go-tsz v0.0.0-20180227144327-03b7d791f4fe h1:VOrqop9SqFzqwZpROEOZpIufuLEUoJ3reNhdOdC9Zzw=
github.com/dgryski/go-tsz v0.0.0-20180227144327-03b7d791f4fe/go.mod h1:ft6P746mYUFQBCsH3OkFBG8FtjLx1XclLMo+9Jh1Yts=
github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
@ -247,8 +245,6 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP
github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
@ -269,8 +265,14 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk=
github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
github.com/freedomkk-qfeng/go-fastping v0.0.0-20160109021039-d7bb493dee3e h1:g8x+P3+xjxt7c53bucQW0ymvj+whjKfCLZH+99UMLS0=
github.com/freedomkk-qfeng/go-fastping v0.0.0-20160109021039-d7bb493dee3e/go.mod h1:UcrAEbxjAhuq5beDj0conKRHGUhBPLkFt8aUmN/jrHY=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gaochao1/gosnmp v0.0.0-20150630013918-783a67a067fd h1:GO2tQE1AIt+P5dwEAVhjH9SlyUcyiN6PJZv8x33QsgU=
github.com/gaochao1/gosnmp v0.0.0-20150630013918-783a67a067fd/go.mod h1:2hQbnwhmIsamIldRzbGxojAjH14ikUBgB4kjk/f6/eI=
github.com/gaochao1/sw v4.0.0+incompatible h1:+f5OVzQHjK8WqJRiSD5y6M3ZANy62VR8o7LFHkQ41iU=
github.com/gaochao1/sw v4.0.0+incompatible/go.mod h1:p/CZIynLtXJo0fzukfdEEYvmwOBi3v8OylDKWJ4rwGo=
github.com/garethr/kubeval v0.0.0-20180821130434-c44f5193dc94/go.mod h1:L8VwozDBY4bGI25r29I6FURZus8xlVo/B7lNOSfre2g=
github.com/garyburd/redigo v1.6.2 h1:yE/pwKCrbLpLpQICzYTeZ7JsTA/C53wFTJHaEtRqniM=
github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
@ -777,8 +779,6 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/open-falcon/rrdlite v0.0.0-20200214140804-bf5829f786ad h1:GXUy5t8CYdaaEj1lRnE22CbHVY1M5h6Rv4kk0PJQc54=
github.com/open-falcon/rrdlite v0.0.0-20200214140804-bf5829f786ad/go.mod h1:pXROoG0iWVnqq4u2Ii97S0Vt9iCTVypshsl9HXsV6cs=
github.com/openconfig/gnmi v0.0.0-20180912164834-33a1865c3029/go.mod h1:t+O9It+LKzfOAhKTT5O0ehDix+MTqbtT0T9t+7zzOvc=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
@ -933,6 +933,8 @@ github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJ
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/sparrc/go-ping v0.0.0-20190613174326-4e5b6552494c h1:gqEdF4VwBu3lTKGHS9rXE9x1/pEaSwCXRLOZRF6qtlw=
github.com/sparrc/go-ping v0.0.0-20190613174326-4e5b6552494c/go.mod h1:eMyUVp6f/5jnzM+3zahzl7q6UXLbgSc3MKg/+ow9QW0=
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
@ -996,8 +998,12 @@ github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKw
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/toolkits/file v0.0.0-20160325033739-a5b3c5147e07 h1:d/VUIMNTk65Xz69htmRPNfjypq2uNRqVsymcXQu6kKk=
github.com/toolkits/file v0.0.0-20160325033739-a5b3c5147e07/go.mod h1:FbXpUxsx5in7z/OrWFDdhYetOy3/VGIJsVHN9G7RUPA=
github.com/toolkits/pkg v1.1.3 h1:cjZMz9hmuTv4v7ivYERA9mWJCLKyr8JMd4S+CL/YzMM=
github.com/toolkits/pkg v1.1.3/go.mod h1:ge83E8FQqUnFk+2wtVtZ8kvbmoSjE1l8FP3f+qmR0fY=
github.com/toolkits/sys v0.0.0-20170615103026-1f33b217ffaf h1:1/LnhAvvotcSWDl1ntwUePzLXcyHjAzZ0Ih0F+kKGZU=
github.com/toolkits/sys v0.0.0-20170615103026-1f33b217ffaf/go.mod h1:GejnAYmB2Pr/2fWKp7OGdd6MzuXvRwClmdQAnvnr++I=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/twmb/murmur3 v1.1.4/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553 h1:DRC1ubdb3ZmyyIeCSTxjZIQAnpLPfKVgYrLETQuOPjo=
@ -1017,8 +1023,6 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/unrolled/render v1.0.3 h1:baO+NG1bZSF2WR4zwh+0bMWauWky7DVrTOfvE2w+aFo=
github.com/unrolled/render v1.0.3/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=

61
sql/n9e_mon.sql
View File

@ -241,7 +241,7 @@ CREATE TABLE `log_collect` (
`created` datetime NOT NULL COMMENT 'created',
`last_updator` varchar(64) NOT NULL DEFAULT '' COMMENT 'last_updator',
`last_updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`whether_attach_one_log_line` tinyint(1) not null default 0,
`whether_attach_one_log_line` tinyint(1) not null default 0,
PRIMARY KEY (`id`),
KEY `idx_nid` (`nid`),
KEY `idx_collect_type` (`collect_type`)
@ -340,65 +340,6 @@ CREATE TABLE `collect_rule` (
KEY `idx_collect_type` (`collect_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT 'collect rule';
CREATE TABLE `aggr_calc` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
`nid` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'nid',
`category` int(1) NOT NULL COMMENT '1 机器 2业务',
`new_metric` varchar(255) NOT NULL DEFAULT '' COMMENT 'new_metric',
`new_step` int(11) NOT NULL DEFAULT '0' COMMENT 'new_step',
`groupby` varchar(255) NOT NULL DEFAULT '' COMMENT 'groupby',
`raw_metrics` text comment 'raw_metrics',
`global_operator` varchar(32) NOT NULL DEFAULT '' COMMENT 'global_operator',
`expression` varchar(255) NOT NULL DEFAULT '' COMMENT 'expression',
`rpn` varchar(255) NOT NULL DEFAULT '' COMMENT 'rpn',
`status` int(1) NOT NULL COMMENT '',
`quota` int(10) NOT NULL COMMENT '',
`comment` varchar(255) NOT NULL DEFAULT '' COMMENT 'comment',
`creator` varchar(64) NOT NULL COMMENT '创建者',
`created` timestamp NOT NULL DEFAULT '1971-01-01 00:00:00' COMMENT 'created',
`last_updator` varchar(64) NOT NULL DEFAULT '',
`last_updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `idx_nid` (`nid`),
KEY `idx_new_metric` (`new_metric`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT 'aggr_calc';
CREATE TABLE `nginx_log_stra` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
`nid` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'nid',
`service` varchar(255) NOT NULL DEFAULT '' COMMENT 'service',
`interval` int(11) NOT NULL DEFAULT '0' COMMENT 'interval',
`domain` varchar(2048) NOT NULL DEFAULT '' COMMENT 'domain',
`url_path_prefix` varchar(2048) NOT NULL DEFAULT '' COMMENT 'url_path_prefix',
`append_tags` varchar(2048) NOT NULL DEFAULT '' COMMENT 'append_tags',
`creator` varchar(64) NOT NULL COMMENT '创建者',
`created` timestamp NOT NULL DEFAULT '1971-01-01 00:00:00' COMMENT 'created',
`last_updator` varchar(64) NOT NULL DEFAULT '',
`last_updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `idx_nid` (`nid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT 'nginx_log_stra';
CREATE TABLE `binlog_stra` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
`nid` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'nid',
`metric` varchar(255) NOT NULL DEFAULT '' COMMENT 'metric',
`interval` int(11) NOT NULL DEFAULT '0' COMMENT 'interval',
`db` varchar(2048) NOT NULL DEFAULT '' COMMENT 'db',
`column_change` varchar(2048) NOT NULL DEFAULT '' COMMENT 'column_change',
`tags_column` varchar(2048) NOT NULL DEFAULT '' COMMENT 'tags_column',
`append_tags` varchar(2048) NOT NULL DEFAULT '' COMMENT 'append_tags',
`func` varchar(255) NOT NULL DEFAULT '' COMMENT 'func',
`sql_type` varchar(255) NOT NULL DEFAULT '' COMMENT 'sql_type',
`value_column` varchar(255) NOT NULL DEFAULT '' COMMENT 'value_column',
`creator` varchar(64) NOT NULL COMMENT '创建者',
`created` timestamp NOT NULL DEFAULT '1971-01-01 00:00:00' COMMENT 'created',
`last_updator` varchar(64) NOT NULL DEFAULT '',
`last_updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `idx_nid` (`nid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT 'binlog_stra';
CREATE TABLE `collect_hist` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
`cid` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'collect id',

2
src/common/address/address.go
View File

@ -2,6 +2,7 @@ package address
import (
"fmt"
"log"
"os"
"path"
"strconv"
@ -118,6 +119,7 @@ func parseConf() {
os.Exit(1)
}
log.Println(c)
mods = c
}

31
src/modules/agent/client/cli.go → src/common/client/cli.go
View File

@ -1,24 +1,28 @@
package client
import (
"bufio"
"io"
"net"
"net/rpc"
"reflect"
"time"
"github.com/didi/nightingale/v4/src/common/address"
"github.com/ugorji/go/codec"
"github.com/toolkits/pkg/logger"
"github.com/toolkits/pkg/net/gobrpc"
"github.com/didi/nightingale/src/common/address"
)
var cli *gobrpc.RPCClient
func getCli() *gobrpc.RPCClient {
func getCli(mod string) *gobrpc.RPCClient {
if cli != nil {
return cli
}
servers := address.GetRPCAddresses("job")
servers := address.GetRPCAddresses(mod)
// detect the fastest server
var (
@ -40,11 +44,22 @@ func getCli() *gobrpc.RPCClient {
continue
}
c := rpc.NewClient(conn)
var bufConn = struct {
io.Closer
*bufio.Reader
*bufio.Writer
}{conn, bufio.NewReader(conn), bufio.NewWriter(conn)}
var mh codec.MsgpackHandle
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
rpcCodec := codec.MsgpackSpecRpc.ClientCodec(bufConn, &mh)
c := rpc.NewClientWithCodec(rpcCodec)
acm[addr] = c
var out string
err = c.Call("Scheduler.Ping", "", &out)
err = c.Call("Server.Ping", "", &out)
if err != nil {
logger.Warningf("ping %s fail: %s", addr, err)
continue
@ -77,9 +92,9 @@ func getCli() *gobrpc.RPCClient {
}
// GetCli 探测所有server端的延迟，自动选择最快的
func GetCli() *gobrpc.RPCClient {
func GetCli(mod string) *gobrpc.RPCClient {
for {
c := getCli()
c := getCli(mod)
if c != nil {
return c
}

48
src/common/compress/compress.go Normal file
View File

@ -0,0 +1,48 @@
package compress
import (
"archive/tar"
"compress/gzip"
"io"
"os"
"strings"
)
func UnCompress(tarFile, dest string) error {
srcFile, err := os.Open(tarFile)
if err != nil {
return err
}
defer srcFile.Close()
gr, err := gzip.NewReader(srcFile)
if err != nil {
return err
}
defer gr.Close()
tr := tar.NewReader(gr)
for {
hdr, err := tr.Next()
if err != nil {
if err == io.EOF {
break
} else {
return err
}
}
filename := dest + "/" + hdr.Name
file, err := createFile(filename)
if err != nil {
return err
}
io.Copy(file, tr)
}
return nil
}
func createFile(name string) (*os.File, error) {
err := os.MkdirAll(string([]rune(name)[0:strings.LastIndex(name, "/")]), 0755)
if err != nil {
return nil, err
}
return os.Create(name)
}

2
src/common/dataobj/judge.go
View File

@ -3,7 +3,7 @@ package dataobj
import (
"strconv"
"github.com/didi/nightingale/src/toolkits/str"
"github.com/didi/nightingale/v4/src/common/str"
gstr "github.com/toolkits/pkg/str"
)

5
src/common/dataobj/snmp.go
View File

@ -9,6 +9,11 @@ const (
COMMON_MODULE = "common"
)
type IPAndSnmpRpcResp struct {
Data []*IPAndSnmp
Msg string
}
type IPAndSnmp struct {
IP string `json:"ip"`
Module string `json:"module"`

4
src/common/dataobj/tsdb.go
View File

@ -5,7 +5,7 @@ import (
"math"
"time"
"github.com/didi/nightingale/src/toolkits/str"
"github.com/didi/nightingale/v4/src/common/str"
)
type JsonFloat float64
@ -95,7 +95,7 @@ func (t *TsdbItem) PrimaryKey() string {
}
func (t *TsdbItem) MD5() string {
return str.MD5(t.Endpoint, t.Metric, str.SortedTags(t.TagsMap))
return str.ToMD5(t.Endpoint, t.Metric, str.SortedTags(t.TagsMap))
}
func (t *TsdbItem) UUID() string {

0
src/toolkits/exit/exit.go → src/common/exit/exit.go
View File

0
src/toolkits/go-tdigest/LICENSE → src/common/go-tdigest/LICENSE
View File

0
src/toolkits/go-tdigest/README.md → src/common/go-tdigest/README.md
View File

0
src/toolkits/go-tdigest/serialization.go → src/common/go-tdigest/serialization.go
View File

0
src/toolkits/go-tdigest/serialization_test.go → src/common/go-tdigest/serialization_test.go
View File

0
src/toolkits/go-tdigest/summary.go → src/common/go-tdigest/summary.go
View File

0
src/toolkits/go-tdigest/summary_test.go → src/common/go-tdigest/summary_test.go
View File

0
src/toolkits/go-tdigest/tdigest.go → src/common/go-tdigest/tdigest.go
View File

0
src/toolkits/go-tdigest/tdigest_test.go → src/common/go-tdigest/tdigest_test.go
View File

0
src/toolkits/i18n/i18n.go → src/common/i18n/i18n.go
View File

28
src/common/loggeri/logger.go
View File

@ -1,28 +0,0 @@
package loggeri
import (
"fmt"
"os"
"github.com/toolkits/pkg/logger"
)
type Config struct {
Dir string `yaml:"dir"`
Level string `yaml:"level"`
KeepHours uint `yaml:"keepHours"`
}
// InitLogger init logger toolkit
func Init(c Config) {
lb, err := logger.NewFileBackend(c.Dir)
if err != nil {
fmt.Println("cannot init logger:", err)
os.Exit(1)
}
lb.SetRotateByHour(true)
lb.SetKeepHours(c.KeepHours)
logger.SetLogging(c.Level, lb)
}

38
src/common/loggeri/loggeri.go Normal file
View File

@ -0,0 +1,38 @@
package loggeri
import (
"fmt"
"os"
"github.com/toolkits/pkg/logger"
)
type Config struct {
Dir string `yaml:"dir"`
Level string `yaml:"level"`
KeepHours uint `yaml:"keepHours"`
Rotatenum int `yaml:"rotatenum"`
Rotatesize uint64 `yaml:"rotatesize"`
}
// InitLogger init logger toolkit
func Init(c Config) {
lb, err := logger.NewFileBackend(c.Dir)
if err != nil {
fmt.Println("cannot init logger:", err)
os.Exit(1)
}
//设置了以小时切换文件，优先使用小时切割文件
if c.KeepHours != 0 {
lb.SetRotateByHour(true)
lb.SetKeepHours(c.KeepHours)
} else if c.Rotatenum != 0 {
lb.Rotate(c.Rotatenum, c.Rotatesize*1024*1024)
} else {
fmt.Println("cannot init logger: KeepHours and Rotatenum is 0")
os.Exit(2)
}
logger.SetLogging(c.Level, lb)
}

76
src/common/ping/ping.go Normal file
View File

@ -0,0 +1,76 @@
package ping
import (
"time"
ping "github.com/sparrc/go-ping"
"github.com/toolkits/pkg/logger"
)
type ipRes struct {
IP string
Good bool
}
func FilterIP(ips []string) []string {
workerNum := 100
worker := make(chan struct{}, workerNum) // 控制 goroutine 并发数
dataChan := make(chan *ipRes, 20000)
done := make(chan struct{}, 1)
goodIps := []string{}
go func() {
defer func() { done <- struct{}{} }()
for d := range dataChan {
if d.Good {
goodIps = append(goodIps, d.IP)
}
}
}()
for _, ip := range ips {
worker <- struct{}{}
go fastPingRtt(ip, 300, worker, dataChan)
}
// 等待所有 goroutine 执行完成
for i := 0; i < workerNum; i++ {
worker <- struct{}{}
}
close(dataChan)
<-done
return goodIps
}
func fastPingRtt(ip string, timeout int, worker chan struct{}, dataChan chan *ipRes) {
defer func() {
<-worker
}()
res := &ipRes{
IP: ip,
Good: goping(ip, timeout),
}
dataChan <- res
}
func goping(ip string, timeout int) bool {
pinger, err := ping.NewPinger(ip)
if err != nil {
panic(err)
}
pinger.SetPrivileged(true)
pinger.Count = 2
pinger.Timeout = time.Duration(timeout) * time.Millisecond
pinger.Interval = time.Duration(timeout) * time.Millisecond
pinger.Run() // blocks until finished
stats := pinger.Statistics() // get send/receive/rtt stats
if stats.PacketsRecv > 0 {
return true
}
logger.Debugf("%+v\n", stats)
return false
}

0
src/toolkits/pools/opentsdb.go → src/common/pools/opentsdb.go
View File

0
src/toolkits/pools/pools.go → src/common/pools/pools.go
View File

106
src/common/report/report.go
View File

@ -2,15 +2,13 @@ package report
import (
"fmt"
"math/rand"
"time"
"github.com/didi/nightingale/src/common/address"
"github.com/didi/nightingale/src/common/identity"
"github.com/didi/nightingale/src/models"
"github.com/didi/nightingale/v4/src/common/client"
"github.com/didi/nightingale/v4/src/common/identity"
"github.com/didi/nightingale/v4/src/models"
"github.com/toolkits/pkg/logger"
"github.com/toolkits/pkg/net/httplib"
)
type ReportSection struct {
@ -26,85 +24,49 @@ type ReportSection struct {
var Config ReportSection
func Init(cfg ReportSection, mod string) {
func Init(cfg ReportSection) {
Config = cfg
addrs := address.GetHTTPAddresses(mod)
t1 := time.NewTicker(time.Duration(Config.Interval) * time.Millisecond)
report(addrs)
for {
<-t1.C
report(addrs)
report()
time.Sleep(time.Duration(Config.Interval) * time.Millisecond)
}
}
type reportRes struct {
Err string `json:"err"`
Dat string `json:"dat"`
}
func report(addrs []string) {
perm := rand.Perm(len(addrs))
for i := range perm {
url := fmt.Sprintf("http://%s/api/hbs/heartbeat", addrs[perm[i]])
ident, _ := identity.GetIdent()
m := map[string]string{
"module": Config.Mod,
"identity": ident,
"rpc_port": Config.RPCPort,
"http_port": Config.HTTPPort,
"remark": Config.Remark,
"region": Config.Region,
}
var body reportRes
err := httplib.Post(url).JSONBodyQuiet(m).SetTimeout(3 * time.Second).ToJSON(&body)
if err != nil {
logger.Errorf("curl %s fail: %v", url, err)
continue
}
if body.Err != "" {
logger.Error(body.Err)
continue
}
func report() {
ident, _ := identity.GetIdent()
instance := models.Instance{
Module: Config.Mod,
Identity: ident,
RPCPort: Config.RPCPort,
HTTPPort: Config.HTTPPort,
Remark: Config.Remark,
Region: Config.Region,
}
var resp string
err := client.GetCli("server").Call("Server.HeartBeat", instance, &resp)
if err != nil {
client.CloseCli()
return
}
if resp != "" {
logger.Errorf("report instance:%+v err:%s", instance, resp)
}
}
type instanceRes struct {
Err string `json:"err"`
Dat []*models.Instance `json:"dat"`
}
func GetAlive(wantedMod string) ([]*models.Instance, error) {
func GetAlive(wantedMod, serverMod string) ([]*models.Instance, error) {
addrs := address.GetHTTPAddresses(serverMod)
perm := rand.Perm(len(addrs))
timeout := 3000
if Config.Timeout != 0 {
timeout = Config.Timeout
var resp *models.InstancesResp
err := client.GetCli("server").Call("Server.InstanceGets", wantedMod, &resp)
if err != nil {
client.CloseCli()
return []*models.Instance{}, fmt.Errorf("get %s instances err:%v", wantedMod, err)
}
var body instanceRes
var err error
for i := range perm {
url := fmt.Sprintf("http://%s/api/hbs/instances?mod=%s&alive=1", addrs[perm[i]], wantedMod)
err = httplib.Get(url).SetTimeout(time.Duration(timeout) * time.Millisecond).ToJSON(&body)
if err != nil {
logger.Warningf("curl %s fail: %v", url, err)
continue
}
if body.Err != "" {
err = fmt.Errorf("curl %s fail: %v", url, body.Err)
logger.Warning(err)
continue
}
if resp.Msg != "" {
return []*models.Instance{}, fmt.Errorf("get %s instances err:%s", wantedMod, resp.Msg)
}
return body.Dat, err
return resp.Data, err
}

51
src/toolkits/slice/slice.go → src/common/slice/slice.go
View File

@ -92,3 +92,54 @@ func Int64In(val int64, slice []int64) bool {
return false
}
// slice set
func Set(s []string) []string {
m := make(map[string]interface{})
for i := 0; i < len(s); i++ {
if strings.TrimSpace(s[i]) == "" {
continue
}
m[s[i]] = 1
}
s2 := []string{}
for k := range m {
s2 = append(s2, k)
}
return s2
}
func InSlice(val string, slice []string) bool {
for i := 0; i < len(slice); i++ {
if slice[i] == val {
return true
}
}
return false
}
func SplitN(m, n int) [][]int {
var res [][]int
if n <= 0 {
return [][]int{[]int{0, m}}
}
for i := 0; i < m; i = i + n {
var start, end int
start = i
end = i + n
if end >= m {
end = m
}
res = append(res, []int{start, end})
}
return res
}

0
src/toolkits/stack/stack.go → src/common/stack/stack.go
View File

0
src/toolkits/stats/counter.go → src/common/stats/counter.go
View File

8
src/toolkits/stats/init.go → src/common/stats/init.go
View File

@ -8,8 +8,8 @@ import (
"path"
"time"
"github.com/didi/nightingale/src/common/address"
"github.com/didi/nightingale/src/common/dataobj"
"github.com/didi/nightingale/v4/src/common/address"
"github.com/didi/nightingale/v4/src/common/dataobj"
"github.com/toolkits/pkg/file"
"github.com/toolkits/pkg/logger"
@ -27,9 +27,9 @@ func Init(prefix string, addr ...string) {
} else if file.IsExist(path.Join(runner.Cwd, "etc", "address.yml")) {
//address.yml 存在，则使用配置文件的地址
newAddr := address.GetHTTPAddresses("agent")
newAddr := address.GetHTTPAddresses("agentd")
if len(newAddr) == 0 {
port := address.GetHTTPPort("agent")
port := address.GetHTTPPort("agentd")
PushUrl = fmt.Sprintf("http://127.0.0.1:%d/v1/push", port)
} else {
PushUrl = fmt.Sprintf("http://%s/v1/push", newAddr[0])

0
src/toolkits/str/checksum.go → src/common/str/checksum.go
View File

5
src/toolkits/str/format.go → src/common/str/format.go
View File

@ -9,9 +9,8 @@ import (
"sync"
"time"
"github.com/toolkits/pkg/str"
"github.com/cespare/xxhash"
"github.com/toolkits/pkg/str"
)
const SEPERATOR = "/"
@ -98,7 +97,7 @@ func XXhash(strs ...string) uint64 {
return xxhash.Sum64(ret.Bytes())
}
func MD5(endpoint string, metric string, tags string) string {
func ToMD5(endpoint string, metric string, tags string) string {
return str.MD5(PK(endpoint, metric, tags))
}

0
src/toolkits/str/parser.go → src/common/str/parser.go
View File

0
src/toolkits/str/slice.go → src/common/str/slice.go
View File

215
src/models/host_register.go Normal file
View File

@ -0,0 +1,215 @@
package models
import (
"encoding/json"
"fmt"
"time"
"github.com/toolkits/pkg/cache"
)
type HostRegisterForm struct {
SN string `json:"sn"`
IP string `json:"ip"`
Ident string `json:"ident"`
Name string `json:"name"`
Cate string `json:"cate"`
UniqKey string `json:"uniqkey"`
Fields map[string]interface{} `json:"fields"`
Digest string `json:"digest"`
}
func (f HostRegisterForm) Validate() error {
if f.IP == "" {
return fmt.Errorf("ip is blank")
}
if f.UniqKey == "" {
return fmt.Errorf("uniqkey is blank")
}
if f.Digest == "" {
return fmt.Errorf("digest is blank")
}
return nil
}
// mapKeyClear map key clear
func MapKeyClear(src map[string]interface{}, save map[string]struct{}) {
var dels []string
for k := range src {
if _, ok := save[k]; !ok {
dels = append(dels, k)
}
}
for i := 0; i < len(dels); i++ {
delete(src, dels[i])
}
}
func HostRegister(f HostRegisterForm) error {
oldFields := make(map[string]interface{}, len(f.Fields))
for k, v := range f.Fields {
oldFields[k] = v
}
uniqValue := ""
if f.UniqKey == "sn" {
uniqValue = f.SN
}
if f.UniqKey == "ip" {
uniqValue = f.IP
}
if f.UniqKey == "ident" {
uniqValue = f.Ident
}
if f.UniqKey == "name" {
uniqValue = f.Name
}
if uniqValue == "" {
return fmt.Errorf("%s is blank", f.UniqKey)
}
cacheKey := "/host/info/" + f.UniqKey + "/" + uniqValue
var val string
if err := cache.Get(cacheKey, &val); err == nil {
if f.Digest == val {
// 说明客户端采集到的各个字段信息并无变化，无需更新DB
return nil
}
} else {
if err.Error() != cache.ErrCacheMiss.Error() {
return fmt.Errorf("get cache:%+v err:%v", f, err)
}
}
host, err := HostGet(f.UniqKey+" = ?", uniqValue)
if err != nil {
return fmt.Errorf("get host:%+v err:%v", f, err)
}
hFixed := map[string]struct{}{
"cpu": struct{}{},
"mem": struct{}{},
"disk": struct{}{},
}
MapKeyClear(f.Fields, hFixed)
if host == nil {
msg := "create host failed"
host, err = HostNew(f.SN, f.IP, f.Ident, f.Name, f.Cate, f.Fields)
if err != nil {
return fmt.Errorf("new host:%+v err:%v", f, err)
}
if host == nil {
return fmt.Errorf("%s, report info:%v", msg, f)
}
} else {
f.Fields["sn"] = f.SN
f.Fields["ip"] = f.IP
f.Fields["ident"] = f.Ident
f.Fields["name"] = f.Name
f.Fields["cate"] = f.Cate
f.Fields["clock"] = time.Now().Unix()
err = host.Update(f.Fields)
if err != nil {
return fmt.Errorf("update host:%+v err:%v", f, err)
}
}
if v, ok := oldFields["tenant"]; ok {
vStr := v.(string)
if vStr != "" {
err = HostUpdateTenant([]int64{host.Id}, vStr)
if err != nil {
return fmt.Errorf("update host:%+v tenant err:%v", f, err)
}
err = ResourceRegister([]Host{*host}, vStr)
if err != nil {
return fmt.Errorf("resource %+v register err:%v", host, err)
}
}
}
if host.Tenant != "" {
// 已经分配给某个租户了，那肯定对应某个resource，需要更新resource的信息
res, err := ResourceGet("uuid=?", fmt.Sprintf("host-%d", host.Id))
if err != nil {
return fmt.Errorf("get resource %v err:%v", host.Id, res)
}
if res == nil {
// 数据不干净，ams里有这个host，而且是已分配状态，但是resource表里没有，重新注册一下
err := ResourceRegister([]Host{*host}, host.Tenant)
if err != nil {
return fmt.Errorf("resource %+v register err:%v", host, err)
}
// 注册完了，重新查询一下试试
res, err = ResourceGet("uuid=?", fmt.Sprintf("host-%d", host.Id))
if err != nil {
return fmt.Errorf("get resource %v err:%v", host.Id, res)
}
if res == nil {
return fmt.Errorf("resource %+v register fail, unknown error", host)
}
}
res.Ident = f.Ident
res.Name = f.Name
res.Cate = f.Cate
MapKeyClear(f.Fields, hFixed)
js, err := json.Marshal(f.Fields)
if err != nil {
return fmt.Errorf("json marshal fields:%v err:%v", f.Fields, err)
}
res.Extend = string(js)
err = res.Update("ident", "name", "cate", "extend")
if err != nil {
return fmt.Errorf("update err:%v", err)
}
}
var objs []HostFieldValue
for k, v := range oldFields {
if k == "tenant" {
continue
}
if _, ok := hFixed[k]; !ok {
tmp := HostFieldValue{HostId: host.Id, FieldIdent: k, FieldValue: v.(string)}
objs = append(objs, tmp)
}
}
if len(objs) > 0 {
err = HostFieldValuePuts(host.Id, objs)
if err != nil {
return fmt.Errorf("host:%+v FieldValue %+v Puts err:%v", host, objs, err)
}
}
err = cache.Set(cacheKey, f.Digest, cache.DEFAULT)
if err != nil {
return fmt.Errorf("set host:%v cache:%s %v err:%v", f, cacheKey, cache.DEFAULT, err)
}
return nil
}

2
src/models/init.go
View File

@ -4,7 +4,7 @@ import (
"fmt"
"time"
"github.com/didi/nightingale/src/toolkits/i18n"
"github.com/didi/nightingale/v4/src/common/i18n"
"github.com/toolkits/pkg/cache"
)

43
src/models/mon_hbs.go → src/models/instance.go
View File

@ -1,6 +1,15 @@
package models
import "time"
import (
"fmt"
"time"
)
//rpc
type InstancesResp struct {
Data []*Instance
Msg string
}
type Instance struct {
Id int64 `json:"id"`
@ -64,3 +73,35 @@ func DelById(id int64) error {
_, err := DB["hbs"].Where("id=?", id).Delete(new(Instance))
return err
}
func ReportHeartBeat(rev Instance) error {
instance, err := GetInstanceBy(rev.Module, rev.Identity, rev.RPCPort, rev.HTTPPort)
if err != nil {
return fmt.Errorf("get instance:%+v err:%v", rev, err)
}
now := time.Now().Unix()
if instance == nil {
instance = &Instance{
Identity: rev.Identity,
Module: rev.Module,
RPCPort: rev.RPCPort,
HTTPPort: rev.HTTPPort,
Region: rev.Region,
TS: now,
}
err := instance.Add()
if err != nil {
return fmt.Errorf("instance:%+v add err:%v", rev, err)
}
} else {
instance.TS = now
instance.HTTPPort = rev.HTTPPort
instance.Region = rev.Region
err := instance.Update()
if err != nil {
return fmt.Errorf("instance:%+v update err:%v", rev, err)
}
}
return nil
}

33
src/models/ldap.go
View File

@ -7,13 +7,38 @@ import (
"gopkg.in/ldap.v3"
"github.com/toolkits/pkg/logger"
"github.com/didi/nightingale/src/modules/rdb/config"
)
type LDAPSection struct {
DefaultUse bool `yaml:"defaultUse"`
Host string `yaml:"host"`
Port int `yaml:"port"`
BaseDn string `yaml:"baseDn"`
BindUser string `yaml:"bindUser"`
BindPass string `yaml:"bindPass"`
AuthFilter string `yaml:"authFilter"`
Attributes ldapAttributes `yaml:"attributes"`
CoverAttributes bool `yaml:"coverAttributes"`
TLS bool `yaml:"tls"`
StartTLS bool `yaml:"startTLS"`
}
type ldapAttributes struct {
Dispname string `yaml:"dispname"`
Phone string `yaml:"phone"`
Email string `yaml:"email"`
Im string `yaml:"im"`
}
var LDAPConfig LDAPSection
func InitLDAP(conf LDAPSection) {
LDAPConfig = conf
}
func genLdapAttributeSearchList() []string {
var ldapAttributes []string
attrs := config.Config.LDAP.Attributes
attrs := LDAPConfig.Attributes
if attrs.Dispname != "" {
ldapAttributes = append(ldapAttributes, attrs.Dispname)
}
@ -32,7 +57,7 @@ func genLdapAttributeSearchList() []string {
func ldapReq(user, pass string) (*ldap.SearchResult, error) {
var conn *ldap.Conn
var err error
lc := config.Config.LDAP
lc := LDAPConfig
addr := fmt.Sprintf("%s:%d", lc.Host, lc.Port)
if lc.TLS {

4
src/models/mon_aggr.go
View File

@ -8,8 +8,8 @@ import (
"time"
"unicode"
"github.com/didi/nightingale/src/common/dataobj"
"github.com/didi/nightingale/src/toolkits/stack"
"github.com/didi/nightingale/v4/src/common/dataobj"
"github.com/didi/nightingale/v4/src/common/stack"
)
type AggrCalc struct {

109
src/models/mon_collect.go
View File

@ -8,6 +8,7 @@ import (
"sync"
"time"
"github.com/didi/nightingale/v4/src/common/dataobj"
"xorm.io/xorm"
)
@ -191,6 +192,11 @@ type LogCollect struct {
WhetherAttachOneLogLine int `json:"whether_attach_one_log_line" xorm:"'whether_attach_one_log_line'"`
}
type ApiCollectRpcResp struct {
Data []*ApiCollect
Msg string
}
type ApiCollect struct {
Id int64 `json:"id"`
Nid int64 `json:"nid"`
@ -531,6 +537,109 @@ func (a *ApiCollect) Update() error {
return err
}
func GetSnmpCollects(nid int64) ([]*SnmpCollect, error) {
collects := []*SnmpCollect{}
if nid == 0 {
err := DB["mon"].Find(&collects)
return collects, err
}
err := DB["mon"].Where("nid = ?", nid).Find(&collects)
return collects, err
}
func (s *SnmpCollect) Encode() error {
if s.OidType < 1 || s.OidType > 3 {
return fmt.Errorf("oid type not support")
}
if s.OidType == 1 {
//补全oid
s.Module = dataobj.COMMON_MODULE
mib, err := MibGet("module=? and metric=?", s.Module, s.Metric)
if err != nil {
return fmt.Errorf("get mib err:%v", err)
}
s.Oid = mib.Oid
} else {
if s.Module == "" {
return fmt.Errorf("module is null")
}
if s.Oid == "" {
return fmt.Errorf("oid is null")
}
if s.MetricType == "" {
return fmt.Errorf("type is null")
}
}
if s.Metric == "" {
return fmt.Errorf("metric is null")
}
indexes, err := json.Marshal(s.Indexes)
if err != nil {
return fmt.Errorf("encode indexes err:%v", err)
}
s.IndexesStr = string(indexes)
return nil
}
func (s *SnmpCollect) Decode() error {
err := json.Unmarshal([]byte(s.IndexesStr), &s.Indexes)
if err != nil {
return fmt.Errorf("decode indexes err:%v", err)
}
return err
}
func (s *SnmpCollect) Update() error {
session := DB["mon"].NewSession()
defer session.Close()
err := session.Begin()
if err != nil {
return err
}
if _, err = session.Id(s.Id).AllCols().Update(s); err != nil {
session.Rollback()
return err
}
b, err := json.Marshal(s)
if err != nil {
session.Rollback()
return err
}
if err := saveHistory(s.Id, "snmp", "update", s.Creator, string(b), session); err != nil {
session.Rollback()
return err
}
if err = session.Commit(); err != nil {
return err
}
return err
}
func (s *SnmpCollect) GetByOidAndModule() (*SnmpCollect, error) {
collect := new(SnmpCollect)
has, err := DB["mon"].Where("oid = ? and nid = ? and module = ? and metric = ?", s.Oid, s.Nid, s.Module, s.Metric).Get(collect)
if !has {
return nil, err
}
collect.Decode()
return collect, err
}
func CreateCollect(collectType, creator string, collect interface{}, dryRun bool) (err error) {
session := DB["mon"].NewSession()
if err = session.Begin(); err != nil {

8
src/models/mon_collect_rule.go
View File

@ -4,7 +4,8 @@ import (
"encoding/json"
"fmt"
"github.com/didi/nightingale/src/common/dataobj"
"github.com/didi/nightingale/v4/src/common/dataobj"
"xorm.io/xorm"
)
@ -12,6 +13,11 @@ const (
defaultStep = 10
)
type CollectRuleRpcResp struct {
Data []*CollectRule
Msg string
}
type CollectRule struct {
Id int64 `json:"id"`
Nid int64 `json:"nid"`

9
src/models/mon_event.go
View File

@ -4,8 +4,13 @@ import (
"encoding/json"
"strings"
"time"
)
"github.com/didi/nightingale/src/modules/monapi/config"
const ALERT = "alert"
const RECOVERY = "recovery"
var (
EventTypeMap = map[string]string{RECOVERY: "恢复", ALERT: "报警"}
)
type Event struct {
@ -220,7 +225,7 @@ func EventAlertUpgradeUnMarshal(str string) (EventAlertUpgrade, error) {
}
func EventCnt(hashid uint64, stime, etime int64, isUpgrade bool) (int64, error) {
session := DB["mon"].Where("hashid = ? and event_type = ? and etime between ? and ?", hashid, config.ALERT, stime, etime)
session := DB["mon"].Where("hashid = ? and event_type = ? and etime between ? and ?", hashid, ALERT, stime, etime)
if isUpgrade {
return session.In("status", GetFlagsByStatus([]string{STATUS_UPGRADE, STATUS_SEND})).Count(new(Event))

134
src/models/nems_mib.go Normal file
View File

@ -0,0 +1,134 @@
package models
import (
"encoding/json"
"regexp"
"strings"
"time"
"xorm.io/xorm"
)
type Module struct {
// A list of OIDs.
Walk []string `yaml:"walk,omitempty"`
Get []string `yaml:"get,omitempty"`
Metrics []*Metric `yaml:"metrics"`
WalkParams WalkParams `yaml:",inline"`
}
type WalkParams struct {
Version int `yaml:"version,omitempty"`
MaxRepetitions uint8 `yaml:"max_repetitions,omitempty"`
Retries int `yaml:"retries,omitempty"`
Timeout time.Duration `yaml:"timeout,omitempty"`
Auth Auth `yaml:"auth,omitempty"`
}
type Metric struct {
Name string `yaml:"name"`
Oid string `yaml:"oid"`
Type string `yaml:"type"`
Help string `yaml:"help"`
Indexes []*Index `yaml:"indexes,omitempty"`
Lookups []*Lookup `yaml:"lookups,omitempty"`
RegexpExtracts map[string][]RegexpExtract `yaml:"regex_extracts,omitempty"`
EnumValues map[int]string `yaml:"enum_values,omitempty"`
}
type RegexpExtract struct {
Value string `yaml:"value"`
Regex Regexp `yaml:"regex"`
}
// Regexp encapsulates a regexp.Regexp and makes it YAML marshalable.
type Regexp struct {
*regexp.Regexp
}
type Mib struct {
Id int64 `json:"id"`
Module string `json:"module"`
Metric string `json:"metric"`
Oid string `json:"oid"`
Mtype string `json:"mtype"` //gauge,counter
EnumValues string `json:"enum_values"`
Indexes string `json:"indexes"`
Note string `json:"note"`
}
func NewMib(module string, m *Metric) *Mib {
enumValues, _ := json.Marshal(m.EnumValues)
indexes, _ := json.Marshal(m.Indexes)
mib := &Mib{
Module: module,
Metric: m.Name,
Oid: m.Oid,
Mtype: m.Type,
EnumValues: string(enumValues),
Indexes: string(indexes),
Note: m.Help,
}
return mib
}
func (m *Mib) Save() error {
_, err := DB["nems"].InsertOne(m)
return err
}
func MibDel(id int64) error {
_, err := DB["nems"].Where("id=?", id).Delete(new(Mib))
return err
}
func MibTotal(query string) (int64, error) {
return buildMibWhere(query).Count()
}
func MibGet(where string, args ...interface{}) (*Mib, error) {
var obj Mib
has, err := DB["nems"].Where(where, args...).Get(&obj)
if !has {
return nil, err
}
return &obj, err
}
func MibGets(where string, args ...interface{}) ([]Mib, error) {
var objs []Mib
err := DB["nems"].Where(where, args...).Find(&objs)
return objs, err
}
func MibGetsGroupBy(group string, where string, args ...interface{}) ([]Mib, error) {
var objs []Mib
var err error
if where == "" {
err = DB["nems"].GroupBy(group).Find(&objs)
} else {
err = DB["nems"].Where(where, args...).GroupBy(group).Find(&objs)
}
return objs, err
}
func MibGetsByQuery(query string, limit, offset int) ([]Mib, error) {
session := buildMibWhere(query)
var objs []Mib
err := session.Limit(limit, offset).Find(&objs)
return objs, err
}
func buildMibWhere(query string) *xorm.Session {
session := DB["nems"].Table(new(Mib))
if query != "" {
arr := strings.Fields(query)
for i := 0; i < len(arr); i++ {
q := "%" + arr[i] + "%"
session = session.Where("module like ? or oid like ? or metric like ?", q, q, q)
}
}
return session
}

298
src/models/nems_nethw.go Normal file
View File

@ -0,0 +1,298 @@
package models
import (
"encoding/json"
"strings"
"github.com/toolkits/pkg/logger"
"github.com/toolkits/pkg/str"
"xorm.io/xorm"
)
type NetworkHardwareRpcResp struct {
Data []*NetworkHardware
Msg string
}
type NetworkHardware struct {
Id int64 `json:"id"`
SN string `json:"sn" xorm:"sn"`
IP string `json:"ip" xorm:"ip"`
Name string `json:"name"`
Note string `json:"note"`
Cate string `json:"cate"`
SnmpVersion string `json:"snmp_version"`
Auth string `json:"auth"`
Region string `json:"region"`
Info string `json:"info"`
Tenant string `json:"tenant"`
Uptime int64 `json:"uptime"`
}
func MakeNetworkHardware(ip, cate, version, auth, region, note string) *NetworkHardware {
obj := &NetworkHardware{
IP: ip,
SnmpVersion: version,
Auth: auth,
Region: region,
Note: note,
Cate: cate,
}
return obj
}
func NetworkHardwareNew(objPtr *NetworkHardware) error {
session := DB["nems"].NewSession()
defer session.Close()
if err := session.Begin(); err != nil {
return err
}
old, err := NetworkHardwareGet("ip=?", objPtr.IP)
if err != nil {
session.Rollback()
return err
}
if old != nil {
session.Rollback()
return nil
}
_, err = session.Insert(objPtr)
if err != nil {
session.Rollback()
return err
}
return session.Commit()
}
func NetworkHardwareGet(where string, args ...interface{}) (*NetworkHardware, error) {
var obj NetworkHardware
has, err := DB["nems"].Where(where, args...).Get(&obj)
if err != nil {
return nil, err
}
if !has {
return nil, nil
}
return &obj, nil
}
func (n *NetworkHardware) Update(cols ...string) error {
session := DB["nems"].NewSession()
defer session.Close()
if err := session.Begin(); err != nil {
return err
}
_, err := session.Where("id=?", n.Id).Cols(cols...).Update(n)
if err != nil {
session.Rollback()
return err
}
return session.Commit()
}
// func (h *Host) Del() error {
// _, err := DB["ams"].Where("id=?", h.Id).Delete(new(Host))
// return err
// }
func (n *NetworkHardware) Del() error {
_, err := DB["nems"].Where("id=?", n.Id).Delete(new(NetworkHardware))
return err
}
func NetworkHardwareCount(where string, args ...interface{}) (int64, error) {
if where != "" {
return DB["nems"].Where(where, args...).Count(new(NetworkHardware))
}
return DB["nems"].Count(new(NetworkHardware))
}
func NetworkHardwareTotal(query string) (int64, error) {
return buildHWWhere(query).Count()
}
func NetworkHardwareList(query string, limit, offset int) ([]NetworkHardware, error) {
session := buildHWWhere(query)
var objs []NetworkHardware
err := session.Limit(limit, offset).OrderBy("id desc").Find(&objs)
return objs, err
}
func buildHWWhere(query string) *xorm.Session {
session := DB["nems"].Table(new(NetworkHardware))
if query != "" {
arr := strings.Fields(query)
for i := 0; i < len(arr); i++ {
q := "%" + arr[i] + "%"
session = session.Where("cate = ? or ip like ? or name like ? or note like ?", arr[i], q, q, q)
}
}
return session
}
func NetworkHardwareDel(id int64) error {
session := DB["nems"].NewSession()
defer session.Close()
if err := session.Begin(); err != nil {
return err
}
var obj NetworkHardware
has, err := session.Where("id=?", id).Get(&obj)
if err != nil {
session.Rollback()
return err
}
if !has {
return err
}
_, err = session.Where("id=?", id).Delete(new(NetworkHardware))
if err != nil {
session.Rollback()
return err
}
return session.Commit()
}
// ResourceRegister 资源分配给某个租户的时候调用
func NetworkHardwareResourceRegister(hws []*NetworkHardware, tenant string) error {
count := len(hws)
for i := 0; i < count; i++ {
uuid := hws[i].SN
res, err := ResourceGet("uuid=?", uuid)
if err != nil {
return err
}
if res == nil {
res = &Resource{
UUID: uuid,
Ident: hws[i].IP,
Name: hws[i].Name,
Cate: hws[i].Cate,
Tenant: tenant,
}
// 如果host加个字段，并且要放到extend里，这里要改
fields := map[string]interface{}{
"region": hws[i].Region,
}
js, err := json.Marshal(fields)
if err != nil {
return err
}
res.Extend = string(js)
err = res.Save()
if err != nil {
return err
}
} else {
if res.Tenant != tenant {
// 之前有归属，如果归属发生变化，解除之前的挂载关系
err = NodeResourceUnbindByRids([]int64{res.Id})
if err != nil {
return err
}
}
res.Ident = hws[i].IP
res.Name = hws[i].Name
res.Cate = hws[i].Cate
res.Tenant = tenant
fields := map[string]interface{}{
"region": hws[i].Region,
}
js, err := json.Marshal(fields)
if err != nil {
return err
}
res.Extend = string(js)
err = res.Update("ident", "name", "cate", "extend", "tenant")
if err != nil {
return err
}
}
}
return nil
}
// NwSearch 普通用户查询
func NwSearch(batch, field string) ([]NetworkHardware, error) {
arr := str.ParseLines(strings.Replace(batch, ",", "\n", -1))
if len(arr) == 0 {
return []NetworkHardware{}, nil
}
var objs []NetworkHardware
err := DB["nems"].Table("network_hardware").In(field, arr).Find(&objs)
return objs, err
}
func NwTotalForAdmin(tenant, query, batch, field string) (int64, error) {
return buildNwWhere(tenant, query, batch, field).Count()
}
func NwGetsForAdmin(tenant, query, batch, field string, limit, offset int) ([]NetworkHardware, error) {
var objs []NetworkHardware
err := buildNwWhere(tenant, query, batch, field).Limit(limit, offset).Find(&objs)
return objs, err
}
func buildNwWhere(tenant, query, batch, field string) *xorm.Session {
session := DB["nems"].Table("network_hardware").OrderBy("id")
if tenant == "0" {
session = session.Where("tenant=?", "")
} else if tenant != "" {
session = session.Where("tenant=?", tenant)
}
if batch == "" && query != "" {
arr := strings.Fields(query)
for i := 0; i < len(arr); i++ {
q := "%" + arr[i] + "%"
session = session.Where("cate=? or sn=? or ip like ? or name like ? or note like ?", arr[i], arr[i], q, q, q)
}
}
if batch != "" {
arr := str.ParseLines(strings.Replace(batch, ",", "\n", -1))
if len(arr) > 0 {
session = session.In(field, arr)
}
}
return session
}
func GetHardwareInfoBy(ips []string) []*NetworkHardware {
var hws []*NetworkHardware
for _, ip := range ips {
hw, err := NetworkHardwareGet("ip=?", ip)
if err != nil {
logger.Error(err)
continue
}
hws = append(hws, hw)
}
return hws
}

4
src/models/node.go
View File

@ -6,7 +6,6 @@ import (
"time"
"github.com/toolkits/pkg/slice"
"github.com/toolkits/pkg/str"
)
@ -591,7 +590,8 @@ func GetLeafNidsForMon(nid int64, exclNid []int64) ([]int64, error) {
if node == nil {
// 节点已经被删了，相关的告警策略也删除
StraDelByNid(nid)
// todo 逻辑需要优化
//StraDelByNid(nid)
return []int64{}, nil
}

0
src/models/node_field_value.go → src/models/node_cate_field_value.go
View File

2
src/models/node_resource.go
View File

@ -3,7 +3,7 @@ package models
import (
"fmt"
"github.com/didi/nightingale/src/toolkits/slice"
"github.com/didi/nightingale/v4/src/common/slice"
)
type NodeResource struct {

1
src/models/node_role.go
View File

@ -34,7 +34,6 @@ func NodeRoleDel(nodeId, roleId int64, username string) error {
return err
}
// RoleIdsBindingUsername
func RoleIdsBindingUsername(username string, nids []int64) ([]int64, error) {
var ids []int64
err := DB["rdb"].Table("node_role").Where("username=?", username).In("node_id", nids).Select("role_id").Find(&ids)

0
src/models/captcha.go → src/models/rdb_captcha.go
View File

0
src/models/configs.go → src/models/rdb_configs.go
View File

0
src/models/session.go → src/models/rdb_session.go
View File

0
src/models/role_operation.go → src/models/role_opration.go
View File

15
src/models/user.go
View File

@ -7,15 +7,12 @@ import (
"strings"
"time"
"github.com/toolkits/pkg/slice"
"github.com/toolkits/pkg/cache"
"github.com/toolkits/pkg/errors"
"github.com/toolkits/pkg/logger"
"github.com/toolkits/pkg/slice"
"github.com/toolkits/pkg/str"
"gopkg.in/ldap.v3"
"github.com/didi/nightingale/src/modules/rdb/config"
)
const (
@ -111,7 +108,7 @@ func (u *User) Validate() error {
}
func (u *User) CopyLdapAttr(sr *ldap.SearchResult) {
attrs := config.Config.LDAP.Attributes
attrs := LDAPConfig.Attributes
if attrs.Dispname != "" {
u.Dispname = sr.Entries[0].GetAttributeValue(attrs.Dispname)
}
@ -173,7 +170,7 @@ func LdapLogin(username, pass string) (*User, error) {
user.CopyLdapAttr(sr)
if has {
if config.Config.LDAP.CoverAttributes {
if LDAPConfig.CoverAttributes {
_, err := DB["rdb"].Where("id=?", user.Id).Update(user)
return &user, err
} else {
@ -710,10 +707,10 @@ func UsersGet(where string, args ...interface{}) ([]User, error) {
return objs, nil
}
func (u *User) PermByNode(node *Node) ([]string, error) {
func (u *User) PermByNode(node *Node, localOpsList []string) ([]string, error) {
// 我是超管，自然有权限
if u.IsRoot == 1 {
return config.LocalOpsList, nil
return localOpsList, nil
}
// 我是path上游的某个admin，自然有权限
@ -729,7 +726,7 @@ func (u *User) PermByNode(node *Node) ([]string, error) {
if yes, err := NodesAdminExists(nodeIds, u.Id); err != nil {
return nil, err
} else if yes {
return config.LocalOpsList, nil
return localOpsList, nil
}
if roleIds, err := RoleIdsBindingUsername(u.Username, nodeIds); err != nil {

0
src/models/stats.go → src/models/user_stats.go
View File

81
src/modules/agent/stra/cron.go
View File

@ -1,81 +0,0 @@
package stra
import (
"fmt"
"math/rand"
"time"
"github.com/toolkits/pkg/logger"
"github.com/toolkits/pkg/net/httplib"
"github.com/didi/nightingale/src/common/address"
"github.com/didi/nightingale/src/models"
"github.com/didi/nightingale/src/modules/agent/config"
)
func GetCollects() {
if !config.Config.Stra.Enable {
return
}
detect()
go loopDetect()
}
func loopDetect() {
t1 := time.NewTicker(time.Duration(config.Config.Stra.Interval) * time.Second)
for {
<-t1.C
detect()
}
}
func detect() {
c, err := GetCollectsRetry()
if err != nil {
logger.Errorf("get collect err:%v", err)
return
}
Collect.Update(&c)
}
type CollectResp struct {
Dat models.Collect `json:"dat"`
Err string `json:"err"`
}
func GetCollectsRetry() (models.Collect, error) {
count := len(address.GetHTTPAddresses("monapi"))
var resp CollectResp
var err error
for i := 0; i < count; i++ {
resp, err = getCollects()
if err == nil {
if resp.Err != "" {
err = fmt.Errorf(resp.Err)
continue
}
return resp.Dat, err
}
}
return resp.Dat, err
}
func getCollects() (CollectResp, error) {
addrs := address.GetHTTPAddresses("monapi")
i := rand.Intn(len(addrs))
addr := addrs[i]
var res CollectResp
var err error
url := fmt.Sprintf("http://%s%s%s", addr, config.Config.Stra.Api, config.Endpoint)
err = httplib.Get(url).SetTimeout(time.Duration(config.Config.Stra.Timeout) * time.Millisecond).ToJSON(&res)
if err != nil {
err = fmt.Errorf("get collects from remote:%s failed, error:%v", url, err)
}
return res, err
}

36
src/modules/agent/agent.go → src/modules/agentd/agentd.go
View File

@ -7,22 +7,22 @@ import (
"os/signal"
"syscall"
"github.com/didi/nightingale/src/common/loggeri"
"github.com/didi/nightingale/src/modules/agent/cache"
"github.com/didi/nightingale/src/modules/agent/config"
"github.com/didi/nightingale/src/modules/agent/http"
"github.com/didi/nightingale/src/modules/agent/log/worker"
"github.com/didi/nightingale/src/modules/agent/report"
"github.com/didi/nightingale/src/modules/agent/statsd"
"github.com/didi/nightingale/src/modules/agent/stra"
"github.com/didi/nightingale/src/modules/agent/sys"
"github.com/didi/nightingale/src/modules/agent/sys/funcs"
"github.com/didi/nightingale/src/modules/agent/sys/plugins"
"github.com/didi/nightingale/src/modules/agent/sys/ports"
"github.com/didi/nightingale/src/modules/agent/sys/procs"
"github.com/didi/nightingale/src/modules/agent/timer"
"github.com/didi/nightingale/src/modules/agent/udp"
"github.com/didi/nightingale/src/toolkits/stats"
"github.com/didi/nightingale/v4/src/common/loggeri"
"github.com/didi/nightingale/v4/src/common/stats"
"github.com/didi/nightingale/v4/src/modules/agentd/cache"
"github.com/didi/nightingale/v4/src/modules/agentd/config"
"github.com/didi/nightingale/v4/src/modules/agentd/http"
"github.com/didi/nightingale/v4/src/modules/agentd/log/worker"
"github.com/didi/nightingale/v4/src/modules/agentd/report"
"github.com/didi/nightingale/v4/src/modules/agentd/statsd"
"github.com/didi/nightingale/v4/src/modules/agentd/stra"
"github.com/didi/nightingale/v4/src/modules/agentd/sys"
"github.com/didi/nightingale/v4/src/modules/agentd/sys/funcs"
"github.com/didi/nightingale/v4/src/modules/agentd/sys/plugins"
"github.com/didi/nightingale/v4/src/modules/agentd/sys/ports"
"github.com/didi/nightingale/v4/src/modules/agentd/sys/procs"
"github.com/didi/nightingale/v4/src/modules/agentd/timer"
"github.com/didi/nightingale/v4/src/modules/agentd/udp"
"github.com/toolkits/pkg/logger"
"github.com/toolkits/pkg/runner"
@ -61,7 +61,7 @@ func main() {
parseConf()
loggeri.Init(config.Config.Logger)
stats.Init("agent")
stats.Init("agentd")
if err := report.GatherBase(); err != nil {
fmt.Println("gatherBase fail: ", err)
@ -143,5 +143,5 @@ func endingProc() {
logger.Close()
http.Shutdown()
fmt.Println("portal stopped successfully")
fmt.Println("agentd stopped successfully")
}

2
src/modules/agent/cache/cache.go → src/modules/agentd/cache/cache.go vendored
View File

@ -4,7 +4,7 @@ import (
"sync"
"time"
"github.com/didi/nightingale/src/common/dataobj"
"github.com/didi/nightingale/v4/src/common/dataobj"
)
var MetricHistory *History

12
src/modules/agent/config/config.go → src/modules/agentd/config/config.go
View File

@ -6,12 +6,12 @@ import (
"strings"
"time"
"github.com/didi/nightingale/v4/src/common/identity"
"github.com/didi/nightingale/v4/src/common/loggeri"
"github.com/didi/nightingale/v4/src/modules/agentd/sys"
"github.com/spf13/viper"
"github.com/toolkits/pkg/file"
"github.com/didi/nightingale/src/common/identity"
"github.com/didi/nightingale/src/common/loggeri"
"github.com/didi/nightingale/src/modules/agent/sys"
)
type ConfigT struct {
@ -166,12 +166,12 @@ func Parse() error {
}
func getYmlFile() string {
yml := "etc/agent.local.yml"
yml := "etc/agentd.local.yml"
if file.IsExist(yml) {
return yml
}
yml = "etc/agent.yml"
yml = "etc/agentd.yml"
if file.IsExist(yml) {
return yml
}

0
src/modules/agent/core/clients.go → src/modules/agentd/core/clients.go
View File

2
src/modules/agent/core/common.go → src/modules/agentd/core/common.go
View File

@ -3,7 +3,7 @@ package core
import (
"strings"
"github.com/didi/nightingale/src/common/dataobj"
"github.com/didi/nightingale/v4/src/common/dataobj"
)
func NewMetricValue(metric string, val interface{}, dataType string, tags ...string) *dataobj.MetricValue {

11
src/modules/agent/client/meta.go → src/modules/agentd/core/meta.go
View File

@ -1,18 +1,21 @@
package client
package core
import (
"fmt"
"github.com/toolkits/pkg/logger"
"github.com/didi/nightingale/v4/src/common/client"
"github.com/didi/nightingale/v4/src/common/dataobj"
"github.com/didi/nightingale/src/common/dataobj"
"github.com/toolkits/pkg/logger"
)
// Meta 从Server端获取任务元信息
func Meta(id int64) (script string, args string, account string, err error) {
var resp dataobj.TaskMetaResponse
err = GetCli().Call("Scheduler.GetTaskMeta", id, &resp)
err = client.GetCli("server").Call("Server.GetTaskMeta", id, &resp)
if err != nil {
logger.Error("rpc call Server.GetTaskMeta get error: ", err)
client.CloseCli()
return
}

14
src/modules/agent/core/push.go → src/modules/agentd/core/push.go
View File

@ -10,13 +10,13 @@ import (
"reflect"
"time"
"github.com/didi/nightingale/v4/src/common/address"
"github.com/didi/nightingale/v4/src/common/dataobj"
"github.com/didi/nightingale/v4/src/modules/agentd/cache"
"github.com/didi/nightingale/v4/src/modules/agentd/config"
"github.com/toolkits/pkg/logger"
"github.com/ugorji/go/codec"
"github.com/didi/nightingale/src/common/address"
"github.com/didi/nightingale/src/common/dataobj"
"github.com/didi/nightingale/src/modules/agent/cache"
"github.com/didi/nightingale/src/modules/agent/config"
)
func Push(metricItems []*dataobj.MetricValue) error {
@ -52,7 +52,7 @@ func Push(metricItems []*dataobj.MetricValue) error {
items = append(items, item)
}
addrs := address.GetRPCAddresses("transfer")
addrs := address.GetRPCAddresses("server")
count := len(addrs)
retry := 0
for {
@ -106,7 +106,7 @@ func rpcCall(addr string, items []*dataobj.MetricValue) (dataobj.TransferResp, e
done := make(chan error, 1)
go func() {
err := client.Call("Transfer.Push", items, &reply)
err := client.Call("Server.Push", items, &reply)
done <- err
}()

8
src/modules/agent/http/http_server.go → src/modules/agentd/http/http_server.go
View File

@ -7,10 +7,10 @@ import (
"os"
"time"
"github.com/gin-gonic/gin"
"github.com/didi/nightingale/v4/src/common/address"
"github.com/didi/nightingale/v4/src/common/middleware"
"github.com/didi/nightingale/src/common/address"
"github.com/didi/nightingale/src/common/middleware"
"github.com/gin-gonic/gin"
)
var srv = &http.Server{
@ -30,7 +30,7 @@ func Start() {
Config(r)
srv.Addr = address.GetHTTPListen("agent")
srv.Addr = address.GetHTTPListen("agentd")
srv.Handler = r
go func() {

0
src/modules/agent/http/router.go → src/modules/agentd/http/router.go
View File

12
src/modules/agent/http/router_collector.go → src/modules/agentd/http/router_collector.go
View File

@ -1,14 +1,14 @@
package http
import (
"github.com/didi/nightingale/v4/src/common/dataobj"
"github.com/didi/nightingale/v4/src/modules/agentd/core"
"github.com/didi/nightingale/v4/src/modules/agentd/log/strategy"
"github.com/didi/nightingale/v4/src/modules/agentd/log/worker"
"github.com/didi/nightingale/v4/src/modules/agentd/stra"
"github.com/gin-gonic/gin"
"github.com/toolkits/pkg/errors"
"github.com/didi/nightingale/src/common/dataobj"
"github.com/didi/nightingale/src/modules/agent/core"
"github.com/didi/nightingale/src/modules/agent/log/strategy"
"github.com/didi/nightingale/src/modules/agent/log/worker"
"github.com/didi/nightingale/src/modules/agent/stra"
)
func pushData(c *gin.Context) {

3
src/modules/agent/http/router_endpoint.go → src/modules/agentd/http/router_endpoint.go
View File

@ -1,7 +1,8 @@
package http
import (
"github.com/didi/nightingale/src/modules/agent/config"
"github.com/didi/nightingale/v4/src/modules/agentd/config"
"github.com/gin-gonic/gin"
)

6
src/modules/agent/http/router_executor.go → src/modules/agentd/http/router_executor.go
View File

@ -6,11 +6,11 @@ import (
"path"
"strconv"
"github.com/didi/nightingale/v4/src/modules/agentd/config"
"github.com/didi/nightingale/v4/src/modules/agentd/timer"
"github.com/gin-gonic/gin"
"github.com/toolkits/pkg/file"
"github.com/didi/nightingale/src/modules/agent/config"
"github.com/didi/nightingale/src/modules/agent/timer"
)
func output(idstr string, typ string) (string, error) {

0
src/modules/agent/http/router_funcs.go → src/modules/agentd/http/router_funcs.go
View File

0
src/modules/agent/http/router_health.go → src/modules/agentd/http/router_health.go
View File

0
src/modules/agent/log/reader/reader.go → src/modules/agentd/log/reader/reader.go
View File

0
src/modules/agent/log/reader/reader_test.go → src/modules/agentd/log/reader/reader_test.go
View File

0
src/modules/agent/log/reader/reader_util.go → src/modules/agentd/log/reader/reader_util.go
View File

0
src/modules/agent/log/reader/reader_util_test.go → src/modules/agentd/log/reader/reader_util_test.go
View File

Some files were not shown because too many files have changed in this diff Show More