From 1807ff35387a5d44379ba801e392a0d9215691a6 Mon Sep 17 00:00:00 2001 From: shiziyuan9527 Date: Wed, 14 Apr 2021 12:02:21 +0800 Subject: [PATCH] =?UTF-8?q?feat(=E6=80=A7=E8=83=BD=E6=B5=8B=E8=AF=95):=20?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=8A=A5=E5=91=8A=E9=A1=B5=E9=9D=A2=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E7=9B=91=E6=8E=A7Tab?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../performance/base/MonitorStatus.java | 5 + .../controller/MetricQueryController.java | 21 ++ .../controller/request/MetricDataRequest.java | 12 + .../controller/request/MetricQuery.java | 19 ++ .../controller/request/MetricRequest.java | 16 ++ .../performance/dto/MetricData.java | 16 ++ .../metersphere/performance/dto/Monitor.java | 16 ++ .../service/MetricQueryService.java | 187 ++++++++++++++ .../service/PerformanceTestService.java | 40 ++- .../report/PerformanceReportView.vue | 5 + .../report/components/MonitorCard.vue | 240 ++++++++++++++++++ .../test/components/EditMonitor.vue | 97 +------ 12 files changed, 588 insertions(+), 86 deletions(-) create mode 100644 backend/src/main/java/io/metersphere/performance/base/MonitorStatus.java create mode 100644 backend/src/main/java/io/metersphere/performance/controller/MetricQueryController.java create mode 100644 backend/src/main/java/io/metersphere/performance/controller/request/MetricDataRequest.java create mode 100644 backend/src/main/java/io/metersphere/performance/controller/request/MetricQuery.java create mode 100644 backend/src/main/java/io/metersphere/performance/controller/request/MetricRequest.java create mode 100644 backend/src/main/java/io/metersphere/performance/dto/MetricData.java create mode 100644 backend/src/main/java/io/metersphere/performance/dto/Monitor.java create mode 100644 backend/src/main/java/io/metersphere/performance/service/MetricQueryService.java create mode 100644 frontend/src/business/components/performance/report/components/MonitorCard.vue diff --git a/backend/src/main/java/io/metersphere/performance/base/MonitorStatus.java b/backend/src/main/java/io/metersphere/performance/base/MonitorStatus.java new file mode 100644 index 0000000000..8eb7adb46d --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/base/MonitorStatus.java @@ -0,0 +1,5 @@ +package io.metersphere.performance.base; + +public enum MonitorStatus { + NOT, NORMAL, ABNORMAL +} diff --git a/backend/src/main/java/io/metersphere/performance/controller/MetricQueryController.java b/backend/src/main/java/io/metersphere/performance/controller/MetricQueryController.java new file mode 100644 index 0000000000..b47bcfe3ec --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/controller/MetricQueryController.java @@ -0,0 +1,21 @@ +package io.metersphere.performance.controller; + +import io.metersphere.performance.dto.MetricData; +import io.metersphere.performance.service.MetricQueryService; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import java.util.List; + +@RestController +@RequestMapping("/metric") +public class MetricQueryController { + + @Resource + private MetricQueryService metricService; + + @GetMapping("/query/{id}") + public List queryMetric(@PathVariable("id") String reportId) { + return metricService.queryMetric(reportId); + } +} diff --git a/backend/src/main/java/io/metersphere/performance/controller/request/MetricDataRequest.java b/backend/src/main/java/io/metersphere/performance/controller/request/MetricDataRequest.java new file mode 100644 index 0000000000..4ab963a490 --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/controller/request/MetricDataRequest.java @@ -0,0 +1,12 @@ +package io.metersphere.performance.controller.request; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class MetricDataRequest { + private String seriesName; + private String promQL; + private String instance; +} diff --git a/backend/src/main/java/io/metersphere/performance/controller/request/MetricQuery.java b/backend/src/main/java/io/metersphere/performance/controller/request/MetricQuery.java new file mode 100644 index 0000000000..45287778a3 --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/controller/request/MetricQuery.java @@ -0,0 +1,19 @@ +package io.metersphere.performance.controller.request; + +import java.util.HashMap; +import java.util.Map; + +public class MetricQuery { + + public static Map getMetricQueryMap() { + return new HashMap(16) {{ + // 指标名:promQL + put("cpu", "100 - (avg by (instance) (irate(node_cpu_seconds_total{instance='%1$s', mode='idle'}[1m])) * 100)"); + put("disk", "100 - node_filesystem_free_bytes{instance='%1$s',fstype!~'rootfs|selinuxfs|autofs|rpc_pipefs|tmpfs|udev|none|devpts|sysfs|debugfs|fuse.*'} / node_filesystem_size_bytes{instance='%1$s',fstype!~'rootfs|selinuxfs|autofs|rpc_pipefs|tmpfs|udev|none|devpts|sysfs|debugfs|fuse.*'} * 100"); + put("memory", "(node_memory_MemTotal_bytes{instance='%1$s'} - node_memory_MemFree_bytes{instance='%1$s'}) / node_memory_MemTotal_bytes{instance='%1$s'} * 100"); + put("netIn", "sum by (instance) (irate(node_network_receive_bytes_total{instance='%1$s',device!~'bond.*?|lo'}[1m])/128)"); + put("netOut", "sum by (instance) (irate(node_network_transmit_bytes_total{instance='%1$s',device!~'bond.*?|lo'}[1m])/128)"); + }}; + + } +} diff --git a/backend/src/main/java/io/metersphere/performance/controller/request/MetricRequest.java b/backend/src/main/java/io/metersphere/performance/controller/request/MetricRequest.java new file mode 100644 index 0000000000..6f573bb7bb --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/controller/request/MetricRequest.java @@ -0,0 +1,16 @@ +package io.metersphere.performance.controller.request; + +import lombok.Getter; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public class MetricRequest { + private List metricDataQueries = new ArrayList<>(); + private long startTime; + private long endTime; + private int step = 15; +} diff --git a/backend/src/main/java/io/metersphere/performance/dto/MetricData.java b/backend/src/main/java/io/metersphere/performance/dto/MetricData.java new file mode 100644 index 0000000000..9db817f022 --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/dto/MetricData.java @@ -0,0 +1,16 @@ +package io.metersphere.performance.dto; + +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +public class MetricData { + private String uniqueLabel; + private String seriesName; + private List values; + private List timestamps; + private String instance; +} diff --git a/backend/src/main/java/io/metersphere/performance/dto/Monitor.java b/backend/src/main/java/io/metersphere/performance/dto/Monitor.java new file mode 100644 index 0000000000..81e3e8d3fa --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/dto/Monitor.java @@ -0,0 +1,16 @@ +package io.metersphere.performance.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class Monitor { + private String name; + private String environmentId; + private String environmentName; + private String ip; + private Integer port; + private String description; + private String monitorStatus; +} diff --git a/backend/src/main/java/io/metersphere/performance/service/MetricQueryService.java b/backend/src/main/java/io/metersphere/performance/service/MetricQueryService.java new file mode 100644 index 0000000000..514f2f9673 --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/service/MetricQueryService.java @@ -0,0 +1,187 @@ +package io.metersphere.performance.service; + + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.nacos.client.utils.StringUtils; +import io.metersphere.base.domain.LoadTestReportWithBLOBs; +import io.metersphere.base.domain.LoadTestWithBLOBs; +import io.metersphere.base.mapper.LoadTestMapper; +import io.metersphere.base.mapper.LoadTestReportMapper; +import io.metersphere.commons.exception.MSException; +import io.metersphere.commons.utils.DateUtils; +import io.metersphere.commons.utils.LogUtil; +import io.metersphere.performance.base.ReportTimeInfo; +import io.metersphere.performance.controller.request.MetricDataRequest; +import io.metersphere.performance.controller.request.MetricQuery; +import io.metersphere.performance.controller.request.MetricRequest; +import io.metersphere.performance.dto.MetricData; +import io.metersphere.performance.dto.Monitor; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.client.RestTemplate; + +import javax.annotation.Resource; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.*; + +@Service +@Transactional(rollbackFor = Exception.class) +public class MetricQueryService { + + + private String prometheusHost = "http://192.168.1.8:9090"; + + @Resource + private RestTemplate restTemplate; + @Resource + private LoadTestReportMapper loadTestReportMapper; + @Resource + private LoadTestMapper loadTestMapper; + @Resource + private ReportService reportService; + + + public List queryMetricData(MetricRequest metricRequest) { + List metricDataList = new ArrayList<>(); + long endTime = metricRequest.getEndTime(); + long startTime = metricRequest.getStartTime(); + int step = metricRequest.getStep(); + long reliableEndTime; + if (endTime > System.currentTimeMillis()) { + reliableEndTime = System.currentTimeMillis(); + } else { + reliableEndTime = endTime; + } + + Optional.ofNullable(metricRequest.getMetricDataQueries()).ifPresent(metricDataQueries -> metricDataQueries.forEach(query -> { + String promQL = query.getPromQL(); + promQL = String.format(promQL, query.getInstance()); + if (StringUtils.isEmpty(promQL)) { + MSException.throwException("promQL is null"); + } else { + Optional.of(queryPrometheusMetric(promQL, query.getSeriesName(), startTime, reliableEndTime, step, query.getInstance())).ifPresent(metricDataList::addAll); + } + })); + + return metricDataList; + } + + + private List queryPrometheusMetric(String promQL, String seriesName, long startTime, long endTime, int step, String instance) { + DecimalFormat df = new DecimalFormat("#.###"); + String start = df.format(startTime / 1000.0); + String end = df.format(endTime / 1000.0); + JSONObject response = restTemplate.getForObject(prometheusHost + "/api/v1/query_range?query={promQL}&start={start}&end={end}&step={step}", JSONObject.class, promQL, start, end, step); + return handleResult(seriesName, response, instance); + } + + private List handleResult(String seriesName, JSONObject response, String instance) { + List list = new ArrayList<>(); + + Map> labelMap = new HashMap<>(); + + if (response != null && StringUtils.equals(response.getString("status"), "success")) { + JSONObject data = response.getJSONObject("data"); + JSONArray result = data.getJSONArray("result"); + + if (result.size() > 1) { + result.forEach(rObject -> { + JSONObject resultObject = new JSONObject((Map)rObject); +// JSONObject resultObject = JSONObject.parseObject(rObject.toString()); + JSONObject metrics = resultObject.getJSONObject("metric"); + + if (metrics != null && metrics.size() > 0) { + for (Map.Entry entry : metrics.entrySet()) + labelMap.computeIfAbsent(entry.getKey(), k -> new HashSet<>()).add(entry.getValue().toString()); + } + }); + } + + Optional uniqueLabelKey = labelMap.entrySet().stream().filter(entry -> entry.getValue().size() == result.size()).map(Map.Entry::getKey).findFirst(); + + result.forEach(rObject -> { + MetricData metricData = new MetricData(); + List timestamps = new ArrayList<>(); + List values = new ArrayList<>(); + + JSONObject resultObject = new JSONObject((Map)rObject); + JSONObject metrics = resultObject.getJSONObject("metric"); + JSONArray jsonArray = resultObject.getJSONArray("values"); + jsonArray.forEach(value -> { + JSONArray ja = JSONObject.parseArray(value.toString()); + Double timestamp = ja.getDouble(0); + try { + timestamps.add(DateUtils.getTimeString((long) (timestamp * 1000))); + } catch (Exception e) { + e.printStackTrace(); + } + values.add(ja.getDouble(1)); + }); + + if (CollectionUtils.isNotEmpty(values)) { + metricData.setValues(values); + metricData.setTimestamps(timestamps); + metricData.setSeriesName(seriesName); + metricData.setInstance(instance); + uniqueLabelKey.ifPresent(s -> metricData.setUniqueLabel(metrics.getString(s))); + list.add(metricData); + } + }); + + + } + + return list; + } + + public List queryMetric(String reportId) { + LoadTestReportWithBLOBs report = loadTestReportMapper.selectByPrimaryKey(reportId); + String testId = report.getTestId(); + LoadTestWithBLOBs loadTestWithBLOBs = loadTestMapper.selectByPrimaryKey(testId); + String advancedConfiguration = loadTestWithBLOBs.getAdvancedConfiguration(); + JSONObject jsonObject = JSON.parseObject(advancedConfiguration); + JSONArray monitorParams = jsonObject.getJSONArray("monitorParams"); + if (monitorParams == null) { + return new ArrayList<>(); + } + List list = new ArrayList<>(); + for (int i = 0; i < monitorParams.size(); i++) { + Monitor monitor = monitorParams.getObject(i, Monitor.class); + String instance = monitor.getIp() + ":" + monitor.getPort(); + getRequest(instance, list); + } + + ReportTimeInfo reportTimeInfo = reportService.getReportTimeInfo(reportId); + MetricRequest metricRequest = new MetricRequest(); + metricRequest.setMetricDataQueries(list); + try { + SimpleDateFormat df = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + Date startTime = df.parse(reportTimeInfo.getStartTime()); + Date endTime = df.parse(reportTimeInfo.getEndTime()); + metricRequest.setStartTime(startTime.getTime()); + metricRequest.setEndTime(endTime.getTime()); + } catch (Exception e) { + LogUtil.error(e, e.getMessage()); + e.printStackTrace(); + } + + return queryMetricData(metricRequest); + } + + private void getRequest(String instance, List list) { + Map map = MetricQuery.getMetricQueryMap(); + Set set = map.keySet(); + set.forEach(s -> { + MetricDataRequest request = new MetricDataRequest(); + String promQL = map.get(s); + request.setPromQL(promQL); + request.setSeriesName(s); + request.setInstance(instance); + list.add(request); + }); + } +} diff --git a/backend/src/main/java/io/metersphere/performance/service/PerformanceTestService.java b/backend/src/main/java/io/metersphere/performance/service/PerformanceTestService.java index 8e33726a6f..2f8a530c1a 100644 --- a/backend/src/main/java/io/metersphere/performance/service/PerformanceTestService.java +++ b/backend/src/main/java/io/metersphere/performance/service/PerformanceTestService.java @@ -1,5 +1,7 @@ package io.metersphere.performance.service; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; import io.metersphere.base.domain.*; import io.metersphere.base.mapper.*; import io.metersphere.base.mapper.ext.ExtLoadTestMapper; @@ -17,10 +19,13 @@ import io.metersphere.controller.request.QueryScheduleRequest; import io.metersphere.controller.request.ScheduleRequest; import io.metersphere.dto.DashboardTestDTO; import io.metersphere.dto.LoadTestDTO; +import io.metersphere.dto.NodeDTO; import io.metersphere.dto.ScheduleDao; import io.metersphere.i18n.Translator; import io.metersphere.job.sechedule.PerformanceTestJob; +import io.metersphere.performance.base.MonitorStatus; import io.metersphere.performance.dto.LoadTestExportJmx; +import io.metersphere.performance.dto.Monitor; import io.metersphere.performance.engine.Engine; import io.metersphere.performance.engine.EngineFactory; import io.metersphere.performance.engine.producer.LoadTestProducer; @@ -44,6 +49,7 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @Service @@ -81,6 +87,8 @@ public class PerformanceTestService { private TestResourcePoolMapper testResourcePoolMapper; @Resource private LoadTestProducer loadTestProducer; + @Resource + private TestResourceMapper testResourceMapper; public List list(QueryTestPlanRequest request) { request.setOrders(ServiceUtils.getDefaultOrder(request.getOrders())); @@ -204,6 +212,36 @@ public class PerformanceTestService { } final LoadTestWithBLOBs loadTest = new LoadTestWithBLOBs(); + + String poolId = request.getTestResourcePoolId(); + TestResourceExample testResourceExample = new TestResourceExample(); + testResourceExample.createCriteria().andTestResourcePoolIdEqualTo(poolId); + List testResources = testResourceMapper.selectByExampleWithBLOBs(testResourceExample); + String advancedConfiguration = request.getAdvancedConfiguration(); + JSONObject jsonObject = JSON.parseObject(advancedConfiguration); + List list = new ArrayList<>(); + + if (!CollectionUtils.isEmpty(testResources)) { + AtomicInteger index = new AtomicInteger(1); + testResources.forEach(testResource -> { + String configuration = testResource.getConfiguration(); + NodeDTO nodeDTO = JSON.parseObject(configuration, NodeDTO.class); + Monitor monitor = new Monitor(); + monitor.setName("名称" + index.getAndIncrement()); + monitor.setDescription("默认生成"); + monitor.setIp(nodeDTO.getIp()); + monitor.setPort(9100); + monitor.setMonitorStatus(MonitorStatus.NORMAL.name()); + list.add(monitor); + }); + } + + if (!CollectionUtils.isEmpty(list)) { + jsonObject.put("monitorParams", list); + } + + advancedConfiguration = JSON.toJSONString(jsonObject); + loadTest.setUserId(SessionUtils.getUser().getId()); loadTest.setId(UUID.randomUUID().toString()); loadTest.setName(request.getName()); @@ -212,7 +250,7 @@ public class PerformanceTestService { loadTest.setUpdateTime(System.currentTimeMillis()); loadTest.setTestResourcePoolId(request.getTestResourcePoolId()); loadTest.setLoadConfiguration(request.getLoadConfiguration()); - loadTest.setAdvancedConfiguration(request.getAdvancedConfiguration()); + loadTest.setAdvancedConfiguration(advancedConfiguration); loadTest.setStatus(PerformanceTestStatus.Saved.name()); loadTest.setNum(getNextNum(request.getProjectId())); loadTestMapper.insert(loadTest); diff --git a/frontend/src/business/components/performance/report/PerformanceReportView.vue b/frontend/src/business/components/performance/report/PerformanceReportView.vue index 262d7a277f..75956d7b13 100644 --- a/frontend/src/business/components/performance/report/PerformanceReportView.vue +++ b/frontend/src/business/components/performance/report/PerformanceReportView.vue @@ -81,6 +81,9 @@ + + + @@ -117,11 +120,13 @@ import html2canvas from 'html2canvas'; import MsPerformanceReportExport from "./PerformanceReportExport"; import {Message} from "element-ui"; import SameTestReports from "@/business/components/performance/report/components/SameTestReports"; +import MonitorCard from "@/business/components/performance/report/components/MonitorCard"; export default { name: "PerformanceReportView", components: { + MonitorCard, SameTestReports, MsPerformanceReportExport, MsReportErrorLog, diff --git a/frontend/src/business/components/performance/report/components/MonitorCard.vue b/frontend/src/business/components/performance/report/components/MonitorCard.vue new file mode 100644 index 0000000000..1802cd262f --- /dev/null +++ b/frontend/src/business/components/performance/report/components/MonitorCard.vue @@ -0,0 +1,240 @@ + + + + + diff --git a/frontend/src/business/components/performance/test/components/EditMonitor.vue b/frontend/src/business/components/performance/test/components/EditMonitor.vue index 811851a510..fa6069c707 100644 --- a/frontend/src/business/components/performance/test/components/EditMonitor.vue +++ b/frontend/src/business/components/performance/test/components/EditMonitor.vue @@ -22,49 +22,19 @@ - - - - - - - - - - - -

监控配置

- - - -
- - - - - - - - - - - - - - - - - - - - - - - - -
- + + + + + + + + + + + + @@ -115,19 +85,10 @@ export default { methods: { open(data, index) { this.index = ''; - this.monitorList = [ - { - indicator: '', - expression: '', - } - ]; this.dialogVisible = true; if (data) { const copy = JSON.parse(JSON.stringify(data)); this.form = copy; - if (copy.monitorConfig) { - this.monitorList = JSON.parse(copy.monitorConfig); - } } if (index !== '' && index !== undefined) { this.index = index; @@ -140,13 +101,6 @@ export default { update() { this.$refs.monitorForm.validate(valid => { if (valid) { - this.form.monitorConfig = JSON.stringify(this.monitorList); - // let authConfig = { - // "ip": this.form.ip, - // "username": this.form.username, - // "password": this.form.password, - // }; - // this.form.authConfig = JSON.stringify(authConfig); this.list.splice(this.index, 1, this.form); this.$emit("update:list", this.list); } else { @@ -158,13 +112,6 @@ export default { create() { this.$refs.monitorForm.validate(valid => { if (valid) { - this.form.monitorConfig = JSON.stringify(this.monitorList); - // let authConfig = { - // "ip": this.form.ip, - // "username": this.form.username, - // "password": this.form.password, - // }; - // this.form.authConfig = JSON.stringify(authConfig); this.form.loadTestId = this.testId; this.form.authStatus = CONFIG_TYPE.NOT; this.form.monitorStatus = CONFIG_TYPE.NOT; @@ -176,26 +123,6 @@ export default { }) this.dialogVisible = false; }, - convertConfig() { - let config = []; - if (this.form.monitorConfig) { - config = JSON.parse(this.form.monitorConfig); - } - this.monitorList = config; - }, - addMonitorConfig() { - this.monitorList.push({ - indicator: '', - expression: '' - }); - }, - delMonitorConfig(index) { - if (this.monitorList.length > 1) { - this.monitorList.splice(index, 1); - } else { - this.$warning("不能删除当前节点"); - } - }, change(data) { let env = this.environments.find(env => env.id === data); this.form.environmentName = env ? env.name : "";