增加自建主机web导入，优化导入逻辑

2022-06-19 12:20:42 +08:00 · 2022-06-19 12:20:42 +08:00 · 1a75a63377
parent 0e38bd4891
commit 1a75a63377
18 changed files with 175 additions and 56 deletions
--- a/10
+++ b/10
@ -1,5 +1,5 @@
 help:
-	echo "Read Makefile" && echo "make build" && echo "make push vf=x.x.x vb=x.x.x"
+	echo "Read Makefile" && echo "make build" && echo "make push ver=x.x.x"
 build:
 	cd flask-consul && docker build -t flask-consul:latest .
 	cd vue-consul && docker build -t nginx-consul:latest .
@ -8,13 +8,13 @@ build:
 push:
 	docker login --username=starsliao@163.com registry.cn-shenzhen.aliyuncs.com
 	docker tag nginx-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:latest
-	docker tag nginx-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:${vf}
+	docker tag nginx-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:${ver}
 	docker tag flask-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:latest
-	docker tag flask-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:${vb}
+	docker tag flask-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:${ver}
 	docker push registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:latest
-	docker push registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:${vf}
+	docker push registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:${ver}
 	docker push registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:latest
-	docker push registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:${vb}
+	docker push registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:${ver}

 update:
 	docker-compose pull && docker-compose up -d
--- a/Roadmap.png
+++ b/Roadmap.png
--- a/docs/ECS主机监控.md
+++ b/docs/ECS主机监控.md
@ -32,7 +32,7 @@

 ### 批量导入自建主机脚本

-在项目仓库根目录的`units`目录下：编辑`selfnode-instance.list`，写入监控目标的信息：机房/公司 租户/部门 区域/项目 分组/环境 名称 实例(ip:端口) 系统(linux/windows)，每行一个，空格分隔。
+在项目仓库根目录的`tools`目录下：编辑`selfnode-instance.list`，写入监控目标的信息：机房/公司 租户/部门 区域/项目 分组/环境 名称 实例(ip:端口) 系统(linux/windows)，每行一个，空格分隔。

 **注意：前5个字段组合起来必须唯一，作为一个监控项的ID。即Consul的ServiceID**

--- a/docs/blackbox站点监控.md
+++ b/docs/blackbox站点监控.md
@ -34,7 +34,7 @@

 #### 批量导入脚本

-在项目仓库根目录的`units`目录下：编辑`blackbox-instance.list`，写入监控目标的信息：监控类型，公司/部门，项目，环境，名称，实例url，每行一个，空格分隔。
+在项目仓库根目录的`tools`目录下：编辑`blackbox-instance.list`，写入监控目标的信息：监控类型，公司/部门，项目，环境，名称，实例url，每行一个，空格分隔。

 **注意：前5个字段组合起来必须唯一，作为一个监控项的ID。即Consul的ServiceID**

--- a/flask-consul/units/token_auth.py
+++ b/flask-consul/units/token_auth.py
@ -9,6 +9,11 @@ auth = HTTPTokenAuth()
 def verify_token(token):
    try:
        data = s.loads(token)
-    except:
+    except Exception as e:
+        print("【login】认证异常",e,flush=True)
        return False
    return True
+
+@auth.error_handler
+def unauthorized():
+    return {"code": 50000, "data": f"403：认证异常，请重新登录！"}, 200
--- a/flask-consul/units/upload.py
+++ b/flask-consul/units/upload.py
@ -5,30 +5,54 @@ import sys
 sys.path.append("..")
 from config import consul_token,consul_url

-def importconsul(row):
+def importconsul(row,imptype):
+    try:
+        if imptype == 'blackbox':
            module, company, project, env, name, instance = row
-    headers = {'X-Consul-Token': consul_token}
            data = {
                "id": f"{module}/{company}/{project}/{env}@{name}",
                "name": 'blackbox_exporter',
                "tags": [module],
-            "Meta": {'module': module, 'company': company, 'project': project, 'env': env, 'name': name,
-                     'instance': instance}
+                "Meta": {'module': module, 'company': company, 'project': project,
+                         'env': env, 'name': name,'instance': instance}
            }
+        elif imptype == 'selfnode':
+            vendor,account,region,group,name,instance,os = row
+            sid = f"{vendor}/{account}/{region}/{group}@{name}"
+            ip = instance.split(':')[0]
+            port = instance.split(':')[1]
+            data = {
+                "id": sid,
+                "name": 'selfnode_exporter',
+                'Address': ip,
+                'port': int(port),
+                "tags": [vendor,os],
+                "Meta": {'vendor':vendor,'account':account,'region':region,'group':group,
+                         'name':name,'instance':instance,'os':os},
+                "check": {"tcp": instance,"interval": "60s"}
+            }
+    except Exception as e:
+        print("【import】导入失败",e,flush=True)
+        return {"code": 50000, "data": f"导入内容格式异常！{row}"} 
+    headers = {'X-Consul-Token': consul_token}

    reg = requests.put(f"{consul_url}/agent/service/register", headers=headers, data=json.dumps(data))
    if reg.status_code == 200:
        print({"code": 20000, "data": "增加成功！"},instance,flush=True)
+        return {"code": 20000, "data": "增加成功！"}
    else:
        print({"code": 50000, "data": f'{reg.status_code}:{reg.text}'},instance,flush=True)
+        return {"code": 50000, "data": f'{reg.status_code}:{reg.text}'}

-def read_execl(file_contents):
+def read_execl(file_contents,imptype):
    data = xlrd.open_workbook(file_contents=file_contents, encoding_override="utf-8")
    table = data.sheets()[0]
-    print("开始读取",flush=True)
+    print("【import】开始读取导入文件",flush=True)
    for rownum in range(table.nrows):
        row = table.row_values(rownum)
        if rownum == 0:
            continue
-        importconsul(row)
-    return {"code": 20000, "data": f"导入成功！"}
+        imp = importconsul(row,imptype)
+        if imp['code'] == 50000:
+            return imp
+    return {"code": 20000, "data": f"导入成功！共导入 {rownum} 条数据。"}
--- a/flask-consul/views/blackbox.py
+++ b/flask-consul/views/blackbox.py
@ -6,7 +6,6 @@ from units import token_auth,blackbox_manager
 from werkzeug.datastructures import FileStorage
 from units import upload

-
 blueprint = Blueprint('blackbox',__name__)
 api = Api(blueprint)

@ -21,11 +20,12 @@ parser.add_argument('del_dict',type=dict)
 parser.add_argument('up_dict',type=dict)
 parser.add_argument('file',type=FileStorage, location="files", help="File is wrong.")

-class Blackbox_Upload_Web(Resource):
+class Upload(Resource):
+    @token_auth.auth.login_required
    def post(self):
        file = parser.parse_args().get("file")
        try:
-            return upload.read_execl(file.read())
+            return upload.read_execl(file.read(),'blackbox')
        except Exception as e:
            print("【blackbox】导入失败",e,flush=True)
            return {"code": 50000, "data": f"导入失败！"}
@ -73,4 +73,4 @@ class BlackboxApi(Resource):
 api.add_resource(GetAllList,'/api/blackbox/alllist')
 api.add_resource(BlackboxApi, '/api/blackbox/service')
 api.add_resource(GetConfig,'/api/blackboxcfg/<stype>')
-api.add_resource(Blackbox_Upload_Web,'/api/blackboxcfg/upload_web')
+api.add_resource(Upload,'/api/blackbox/upload')
--- a/flask-consul/views/selfnode.py
+++ b/flask-consul/views/selfnode.py
@ -3,6 +3,8 @@ from flask_restful import reqparse, Resource, Api
 import sys
 sys.path.append("..")
 from units import token_auth,selfnode_manager
+from werkzeug.datastructures import FileStorage
+from units import upload

 blueprint = Blueprint('selfnode',__name__)
 api = Api(blueprint)
@ -18,6 +20,17 @@ parser.add_argument('port',type=str)
 parser.add_argument('os',type=str)
 parser.add_argument('del_dict',type=dict)
 parser.add_argument('up_dict',type=dict)
+parser.add_argument('file',type=FileStorage, location="files", help="File is wrong.")
+
+class Upload(Resource):
+    @token_auth.auth.login_required
+    def post(self):
+        file = parser.parse_args().get("file")
+        try:
+            return upload.read_execl(file.read(),'selfnode')
+        except Exception as e:
+            print("【selfnode】导入失败",e,flush=True)
+            return {"code": 50000, "data": f"导入失败！"}

 class GetAllList(Resource):
    @token_auth.auth.login_required
@ -53,3 +66,4 @@ class SelfnodeApi(Resource):

 api.add_resource(GetAllList,'/api/selfnode/alllist')
 api.add_resource(SelfnodeApi, '/api/selfnode/service')
+api.add_resource(Upload,'/api/selfnode/upload')
--- a/tools/blackbox-input.py
+++ b/tools/blackbox-input.py
--- a/tools/blackbox-instance.list
+++ b/tools/blackbox-instance.list
--- a/tools/docker-build.sh
+++ b/tools/docker-build.sh
--- a/tools/docker-push.sh
+++ b/tools/docker-push.sh
@ -1,15 +1,14 @@
 #!/bin/bash
-vf=0.3.1
-vb=0.3.1
-docker login --username=starsliao@163.com registry.cn-shenzhen.aliyuncs.com
+ver=0.26
+docker login --username=youremail registry.cn-shenzhen.aliyuncs.com

 docker tag nginx-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:latest
-docker tag nginx-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:${vf}
+docker tag nginx-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:${ver}

 docker tag flask-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:latest
-docker tag flask-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:${vb}
+docker tag flask-consul:latest registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:${ver}

 docker push registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:latest
-docker push registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:${vf}
+docker push registry.cn-shenzhen.aliyuncs.com/starsl/nginx-consul:${ver}
 docker push registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:latest
-docker push registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:${vb}
+docker push registry.cn-shenzhen.aliyuncs.com/starsl/flask-consul:${ver}
--- a/tools/selfnode-input.py
+++ b/tools/selfnode-input.py
--- a/tools/selfnode-instance.list
+++ b/tools/selfnode-instance.list
--- a/tools/当前已支持Web导入
+++ b/tools/当前已支持Web导入
--- a/vue-consul/src/api/blackbox.js
+++ b/vue-consul/src/api/blackbox.js
@ -53,11 +53,3 @@ export function getBconfig() {
    method: 'get'
  })
 }
-
-export function upload_web(data) {
-  return request({
-    url: '/api/blackboxcfg/upload_web',
-    method: 'get',
-    data
-  })
-}
--- a/vue-consul/src/views/blackbox/index.vue
+++ b/vue-consul/src/views/blackbox/index.vue
@ -3,17 +3,17 @@
    <el-alert type="success" center close-text="知道了">
      <el-link icon="el-icon-warning" type="success" href="https://github.com/starsliao/ConsulManager/blob/main/docs/blackbox%E7%AB%99%E7%82%B9%E7%9B%91%E6%8E%A7.md" target="_blank">应用场景：如何优雅的使用Consul管理Blackbox站点监控</el-link>
    </el-alert>
-    <div class="filter-container" style="flex: 1;display: flex;align-items: center;height: 100px;">
+    <div class="filter-container" style="flex: 1;display: flex;align-items: center;height: 50px;">
      <el-select v-model="listQuery.module" placeholder="监控类型" clearable collapse-tags style="width: 150px" class="filter-item">
        <el-option v-for="item in module_list" :key="item" :label="item" :value="item" />
      </el-select>
      <el-select v-model="listQuery.company" placeholder="公司部门" clearable style="width: 150px" class="filter-item">
        <el-option v-for="item in company_list" :key="item" :label="item" :value="item" />
      </el-select>
-      <el-select v-model="listQuery.project" filterable placeholder="项目" clearable style="width: 160px" class="filter-item">
+      <el-select v-model="listQuery.project" filterable placeholder="项目" clearable style="width: 150px" class="filter-item">
        <el-option v-for="item in project_list" :key="item" :label="item" :value="item" />
      </el-select>
-      <el-select v-model="listQuery.env" filterable placeholder="环境" clearable style="width: 100px" class="filter-item">
+      <el-select v-model="listQuery.env" filterable placeholder="环境" clearable style="width: 120px" class="filter-item">
        <el-option v-for="item in env_list" :key="item" :label="item" :value="item" />
      </el-select>
      <el-tooltip class="item" effect="light" content="点击清空查询条件" placement="top">
@ -22,11 +22,22 @@
      <el-button class="filter-item" type="primary" icon="el-icon-edit" @click="handleCreate">
        新增
      </el-button>
-      <el-button v-waves :loading="downloadLoading" class="filter-item" type="primary" icon="el-icon-download" @click="handleDownload">
+      <el-button v-waves :loading="downloadLoading" class="filter-item" type="success" icon="el-icon-download" @click="handleDownload">
        导出
      </el-button>
-      <el-upload style="margin-right: 10px;" class="upload-demo" action="/api/blackboxcfg/upload_web" :on-success="success" :on-error="error" accept=".xlsx" :show-file-list="false" multiple :limit="1">
-        <el-button v-waves style="margin-left: 10px;" :loading="downloadLoading" class="filter-item" type="success" icon="el-icon-upload2">
+      <el-upload
+        style="margin-right: 10px;"
+        class="upload-demo"
+        action="/api/blackbox/upload"
+        :headers="myHeaders"
+        :on-success="success"
+        :on-error="error"
+        accept=".xlsx"
+        :before-upload="handleBeforeUpload"
+        :show-file-list="false"
+        :multiple="false"
+      >
+        <el-button v-waves style="margin-left: 10px;" :loading="downloadLoading" class="filter-item" type="warning" icon="el-icon-upload2">
          导入
        </el-button>
      </el-upload>
@ -34,7 +45,7 @@
        批量删除
      </el-button>
      <div style="float: right;margin-left: 10px;">
-        <el-input v-model="iname" prefix-icon="el-icon-search" placeholder="请输入名称或实例进行筛选" clearable style="width:230px" class="filter-item" @input="inameFilter(iname)" />
+        <el-input v-model="iname" prefix-icon="el-icon-search" placeholder="请输入名称或实例进行筛选" clearable style="width:180px" class="filter-item" @input="inameFilter(iname)" />
      </div>
    </div>

@ -103,7 +114,7 @@
          <span slot="label">
            <span class="span-box">
              <span>监控类型</span>
-              <el-tooltip style="diaplay:inline" effect="dark" content="该字段必须和Blackbox配置中的module名称保持一致，如：http_2xx，http_post_2xx，http_post_2xx 等。" placement="top">
+              <el-tooltip style="diaplay:inline" effect="dark" content="该字段必须和Blackbox配置中的module名称保持一致，如：http_2xx，http_post_2xx，tcp_connect 等。" placement="top">
                <i class="el-icon-info" />
              </el-tooltip>
            </span>
@ -178,6 +189,7 @@ export default {
      }
    }
    return {
+      myHeaders: { Authorization: this.$store.getters.token },
      all_list: [],
      pall_list: [],
      iname: '',
@ -264,6 +276,23 @@ export default {
  },

  methods: {
+    handleBeforeUpload(file) {
+      const uploadLimit = 5
+      const uploadTypes = ['xlsx']
+      const filetype = file.name.replace(/.+\./, '')
+      const isRightSize = (file.size || 0) / 1024 / 1024 < uploadLimit
+      if (!isRightSize) {
+        this.$message.error(`文件大小超过${uploadLimit}MB！`)
+        return false
+      }
+      if (uploadTypes.indexOf(filetype.toLowerCase()) === -1) {
+        this.$message.warning({
+          message: '仅支持上传xlsx格式的文件！'
+        })
+        return false
+      }
+      return true
+    },
    success(response) {
      if (response.code === 20000) {
        this.fetchData()
@ -476,7 +505,7 @@ export default {
    handleDownload() {
      this.downloadLoading = true
      import('@/vendor/Export2Excel').then(excel => {
-        const tHeader = ['监控模块', '公司部门', '项目', '环境', '名称', '实例']
+        const tHeader = ['监控模块', '公司部门', '项目', '环境', '名称', '实例(tcp的格式为IP:端口,URL需要以http(s)://开头)']
        const filterVal = ['module', 'company', 'project', 'env', 'name', 'instance']
        const data = this.formatJson(filterVal)
        excel.export_json_to_excel({
--- a/vue-consul/src/views/node-exporter/self.vue
+++ b/vue-consul/src/views/node-exporter/self.vue
@ -1,16 +1,16 @@
 <template>
  <div class="app-container">
-    <div class="filter-container">
+    <div class="filter-container" style="flex: 1;display: flex;align-items: center;height: 50px;">
      <el-select v-model="listQuery.vendor" placeholder="机房/公司" clearable collapse-tags style="width: 150px" class="filter-item">
        <el-option v-for="item in vendor_list" :key="item" :label="item" :value="item" />
      </el-select>
      <el-select v-model="listQuery.account" placeholder="租户/部门" clearable style="width: 150px" class="filter-item">
        <el-option v-for="item in account_list" :key="item" :label="item" :value="item" />
      </el-select>
-      <el-select v-model="listQuery.region" filterable placeholder="区域/项目" clearable style="width: 160px" class="filter-item">
+      <el-select v-model="listQuery.region" filterable placeholder="区域/项目" clearable style="width: 150px" class="filter-item">
        <el-option v-for="item in region_list" :key="item" :label="item" :value="item" />
      </el-select>
-      <el-select v-model="listQuery.group" filterable placeholder="分组/环境" clearable style="width: 100px" class="filter-item">
+      <el-select v-model="listQuery.group" filterable placeholder="分组/环境" clearable style="width: 120px" class="filter-item">
        <el-option v-for="item in group_list" :key="item" :label="item" :value="item" />
      </el-select>
      <el-tooltip class="item" effect="light" content="点击清空查询条件" placement="top">
@ -19,14 +19,30 @@
      <el-button class="filter-item" type="primary" icon="el-icon-edit" @click="handleCreate">
        新增
      </el-button>
-      <el-button v-waves :loading="downloadLoading" class="filter-item" type="primary" icon="el-icon-download" @click="handleDownload">
+      <el-button v-waves :loading="downloadLoading" class="filter-item" type="success" icon="el-icon-download" @click="handleDownload">
        导出
      </el-button>
+      <el-upload
+        style="margin-right: 10px;"
+        class="upload-demo"
+        action="/api/selfnode/upload"
+        :headers="myHeaders"
+        :on-success="success"
+        :on-error="error"
+        accept=".xlsx"
+        :before-upload="handleBeforeUpload"
+        :show-file-list="false"
+        :multiple="false"
+      >
+        <el-button v-waves style="margin-left: 10px;" :loading="downloadLoading" class="filter-item" type="warning" icon="el-icon-upload2">
+          导入
+        </el-button>
+      </el-upload>
      <el-button class="filter-item" type="danger" icon="el-icon-delete" @click="handleDelAll">
        批量删除
      </el-button>
-      <div style="float: right;">
-        <el-input v-model="iname" prefix-icon="el-icon-search" placeholder="请输入名称或实例进行筛选" clearable style="width:230px" class="filter-item" @input="inameFilter(iname)" />
+      <div style="float: right;margin-left: 10px;">
+        <el-input v-model="iname" prefix-icon="el-icon-search" placeholder="请输入名称或实例进行筛选" clearable style="width:180px" class="filter-item" @input="inameFilter(iname)" />
      </div>
    </div>

@ -171,6 +187,7 @@ export default {
      }
    }
    return {
+      myHeaders: { Authorization: this.$store.getters.token },
      all_list: [],
      pall_list: [],
      iname: '',
@ -265,6 +282,45 @@ export default {
  },

  methods: {
+    handleBeforeUpload(file) {
+      const uploadLimit = 5
+      const uploadTypes = ['xlsx']
+      const filetype = file.name.replace(/.+\./, '')
+      const isRightSize = (file.size || 0) / 1024 / 1024 < uploadLimit
+      if (!isRightSize) {
+        this.$message.error(`文件大小超过${uploadLimit}MB！`)
+        return false
+      }
+      if (uploadTypes.indexOf(filetype.toLowerCase()) === -1) {
+        this.$message.warning({
+          message: '仅支持上传xlsx格式的文件！'
+        })
+        return false
+      }
+      return true
+    },
+    success(response) {
+      if (response.code === 20000) {
+        this.fetchData()
+        this.$message({
+          message: response.data,
+          type: 'success'
+        })
+      } else {
+        this.$message({
+          message: response.data,
+          type: 'error'
+        })
+      }
+    },
+    error(response) {
+      if (response.code === 50000) {
+        this.$message({
+          message: response.data,
+          type: 'error'
+        })
+      }
+    },
    inameFilter(iname) {
      if (iname === '') {
        this.handleFilter()
@ -464,7 +520,7 @@ export default {
    handleDownload() {
      this.downloadLoading = true
      import('@/vendor/Export2Excel').then(excel => {
-        const tHeader = ['机房/公司', '租户/部门', '区域/项目', '分组/环境', '名称', '实例', '系统']
+        const tHeader = ['机房/公司', '租户/部门', '区域/项目', '分组/环境', '名称', '实例(IP:端口)', '系统(linux/windows)']
        const filterVal = ['vendor', 'account', 'region', 'group', 'name', 'instance', 'os']
        const data = this.formatJson(filterVal)
        excel.export_json_to_excel({