From 36961210f2d0bef68244eb006ebd5b56a87550bd Mon Sep 17 00:00:00 2001 From: song-tianyang Date: Wed, 22 Dec 2021 13:52:53 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E6=B5=8B=E8=AF=95=E8=AE=A1=E5=88=92?= =?UTF-8?q?=E6=89=A7=E8=A1=8C):=20=E5=A2=9E=E5=8A=A0quartz=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E5=B9=B6=E5=90=88=E5=B9=B61.14=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=91=E7=8E=B0=E7=9A=84=E9=92=88=E5=AF=B9=E5=A4=9A=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E6=B5=8B=E8=AF=95=E8=AE=A1=E5=88=92=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E6=97=B6=E7=9A=84=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加quartz配置并合并1.14版本发现的针对多节点测试计划执行时的代码更改 --- .../api/cache/TestPlanExecuteInfo.java | 69 +++++++++++++--- .../api/cache/TestPlanReportExecuteCatch.java | 7 +- .../job/sechedule/TestPlanTestJob.java | 28 +++---- .../listener/AppStartListener.java | 15 +++- .../TestPlanReportExecuteCheckResultDTO.java | 11 +++ .../track/service/TestPlanReportService.java | 79 +++++++++++++++---- .../src/main/resources/application.properties | 7 +- 7 files changed, 166 insertions(+), 50 deletions(-) create mode 100644 backend/src/main/java/io/metersphere/track/dto/TestPlanReportExecuteCheckResultDTO.java diff --git a/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java b/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java index 295ca7ebdc..6946e887de 100644 --- a/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java +++ b/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java @@ -9,6 +9,7 @@ import io.metersphere.base.mapper.ApiScenarioReportMapper; import io.metersphere.commons.constants.TestPlanApiExecuteStatus; import io.metersphere.commons.constants.TestPlanResourceType; import io.metersphere.commons.utils.CommonBeanFactory; +import io.metersphere.track.dto.TestPlanReportExecuteCheckResultDTO; import io.metersphere.utils.LoggerUtil; import lombok.Getter; import lombok.Setter; @@ -20,6 +21,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * @author song.tianyang @@ -30,17 +32,16 @@ import java.util.Map; public class TestPlanExecuteInfo { private String reportId; private String creator; - private Map apiCaseExecInfo = new HashMap<>(); - private Map apiScenarioCaseExecInfo = new HashMap<>(); - private Map loadCaseExecInfo = new HashMap<>(); + private Map apiCaseExecInfo = new ConcurrentHashMap<>(); + private Map apiScenarioCaseExecInfo = new ConcurrentHashMap<>(); + private Map loadCaseExecInfo = new ConcurrentHashMap<>(); - private Map apiCaseExecuteThreadMap = new HashMap<>(); - private Map apiScenarioThreadMap = new HashMap<>(); - private Map loadCaseReportIdMap = new HashMap<>(); - - private Map apiCaseReportMap = new HashMap<>(); - private Map apiScenarioReportMap = new HashMap<>(); + private Map apiCaseExecuteThreadMap = new ConcurrentHashMap<>(); + private Map apiScenarioThreadMap = new ConcurrentHashMap<>(); + private Map loadCaseReportIdMap = new ConcurrentHashMap<>(); + private Map apiCaseReportMap = new ConcurrentHashMap<>(); + private Map apiScenarioReportMap = new ConcurrentHashMap<>(); private boolean reportDataInDataBase; int lastUnFinishedNumCount = 0; @@ -83,7 +84,8 @@ public class TestPlanExecuteInfo { } } - public synchronized int countUnFinishedNum() { + public synchronized TestPlanReportExecuteCheckResultDTO countUnFinishedNum() { + TestPlanReportExecuteCheckResultDTO executeCheck = new TestPlanReportExecuteCheckResultDTO(); int unFinishedCount = 0; this.isApiCaseAllExecuted = true; @@ -129,8 +131,22 @@ public class TestPlanExecuteInfo { LoggerUtil.info("执行的报告还在队列中，重置超时时间"); lastUnFinishedNumCount = unFinishedCount; lastFinishedNumCountTime = System.currentTimeMillis(); + executeCheck.setFinishedCaseChanged(true); + } else if (unFinishedCount == 0) { + executeCheck.setFinishedCaseChanged(true); + } else { + executeCheck.setFinishedCaseChanged(false); } - return unFinishedCount; + + executeCheck.setTimeOut(false); + if (unFinishedCount > 0) { + //20分钟没有案例执行结果更新，则定位超时 + long nowTime = System.currentTimeMillis(); + if (nowTime - lastFinishedNumCountTime > 1200000) { + executeCheck.setTimeOut(true); + } + } + return executeCheck; } public Map> getExecutedResult() { @@ -228,7 +244,7 @@ public class TestPlanExecuteInfo { this.countUnFinishedNum(); } - public void updateReport(Map apiCaseExecResultInfo, Map apiScenarioCaseExecResultInfo) { + public synchronized void updateReport(Map apiCaseExecResultInfo, Map apiScenarioCaseExecResultInfo) { if (MapUtils.isNotEmpty(apiCaseExecResultInfo)) { this.apiCaseReportMap.putAll(apiCaseExecResultInfo); } @@ -236,6 +252,35 @@ public class TestPlanExecuteInfo { if (MapUtils.isNotEmpty(apiScenarioCaseExecResultInfo)) { this.apiScenarioReportMap.putAll(apiScenarioCaseExecResultInfo); } + } + public Map getRunningApiCaseReportMap() { + //key: reportId, value: testPlanApiCaseId + Map returnMap = new HashMap<>(); + for (Map.Entry entry : apiCaseExecInfo.entrySet()) { + String planCaseId = entry.getKey(); + String status = entry.getValue(); + if (StringUtils.equalsIgnoreCase(status, TestPlanApiExecuteStatus.RUNNING.name())) { + if (apiCaseExecuteThreadMap.containsKey(planCaseId)) { + returnMap.put(apiCaseExecuteThreadMap.get(planCaseId), planCaseId); + } + } + } + return returnMap; + } + + public Map getRunningScenarioReportMap() { + //key: reportId, value: testPlanApiScenarioId + Map returnMap = new HashMap<>(); + for (Map.Entry entry : apiScenarioCaseExecInfo.entrySet()) { + String planScenarioId = entry.getKey(); + String status = entry.getValue(); + if (StringUtils.equalsIgnoreCase(status, TestPlanApiExecuteStatus.RUNNING.name())) { + if (apiScenarioThreadMap.containsKey(planScenarioId)) { + returnMap.put(apiScenarioThreadMap.get(planScenarioId), planScenarioId); + } + } + } + return returnMap; } } \ No newline at end of file diff --git a/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java b/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java index 7971a362de..3f4c680264 100644 --- a/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java +++ b/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java @@ -1,6 +1,7 @@ package io.metersphere.api.cache; import io.metersphere.commons.constants.TestPlanApiExecuteStatus; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +51,11 @@ public class TestPlanReportExecuteCatch { } public synchronized static boolean containsReport(String reportId) { - return testPlanReportMap != null && testPlanReportMap.containsKey(reportId); + if(StringUtils.isEmpty(reportId)){ + return false; + }else { + return testPlanReportMap != null && testPlanReportMap.containsKey(reportId); + } } public synchronized static void updateApiTestPlanExecuteInfo(String reportId, diff --git a/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java b/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java index 1f03186e9c..589fbf9812 100644 --- a/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java +++ b/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java @@ -3,6 +3,7 @@ package io.metersphere.job.sechedule; import io.metersphere.commons.constants.ReportTriggerMode; import io.metersphere.commons.constants.ScheduleGroup; import io.metersphere.commons.utils.CommonBeanFactory; +import io.metersphere.commons.utils.LogUtil; import io.metersphere.track.service.TestPlanService; import org.quartz.*; @@ -16,25 +17,10 @@ import org.quartz.*; public class TestPlanTestJob extends MsScheduleJob { private String projectID; - - // private PerformanceTestService performanceTestService; -// private TestPlanScenarioCaseService testPlanScenarioCaseService; -// private TestPlanApiCaseService testPlanApiCaseService; -// private ApiTestCaseService apiTestCaseService; -// private TestPlanReportService testPlanReportService; -// private TestPlanLoadCaseService testPlanLoadCaseService; private TestPlanService testPlanService; public TestPlanTestJob() { -// this.performanceTestService = CommonBeanFactory.getBean(PerformanceTestService.class); -// this.testPlanScenarioCaseService = CommonBeanFactory.getBean(TestPlanScenarioCaseService.class); -// this.testPlanApiCaseService = CommonBeanFactory.getBean(TestPlanApiCaseService.class); -// this.apiTestCaseService = CommonBeanFactory.getBean(ApiTestCaseService.class); -// this.testPlanReportService = CommonBeanFactory.getBean(TestPlanReportService.class); -// this.testPlanLoadCaseService = CommonBeanFactory.getBean(TestPlanLoadCaseService.class); this.testPlanService = CommonBeanFactory.getBean(TestPlanService.class); - - } /** @@ -63,7 +49,17 @@ public class TestPlanTestJob extends MsScheduleJob { JobDataMap jobDataMap = context.getJobDetail().getJobDataMap(); String config = jobDataMap.getString("config"); - testPlanService.run(this.resourceId, this.projectID, this.userId, ReportTriggerMode.SCHEDULE.name(),config); + String runResourceId = this.resourceId; + String runProjectId = this.projectID; + String runUserId = this.userId; + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + LogUtil.info("Start test_plan_scehdule. test_plan_id:" + runResourceId); + testPlanService.run(runResourceId, runProjectId, runUserId, ReportTriggerMode.SCHEDULE.name(),config); + } + }); + thread.start(); } public static JobKey getJobKey(String testId) { diff --git a/backend/src/main/java/io/metersphere/listener/AppStartListener.java b/backend/src/main/java/io/metersphere/listener/AppStartListener.java index 7385d38b26..e6289d31f3 100644 --- a/backend/src/main/java/io/metersphere/listener/AppStartListener.java +++ b/backend/src/main/java/io/metersphere/listener/AppStartListener.java @@ -68,6 +68,14 @@ public class AppStartListener implements ApplicationListener" + apiCaseIsOk + "; scenario is over ->" + scenarioIsOk + "; performance is over ->" + performanceIsOk); - if (apiCaseIsOk) { testPlanReport.setIsApiCaseExecuting(false); } @@ -1075,12 +1075,16 @@ public class TestPlanReportService { } public void countReport(String planReportId) { - boolean isTimeOut = this.checkTestPlanReportIsTimeOut(planReportId); - if (isTimeOut) { + TestPlanReportExecuteCheckResultDTO checkResult = this.checkTestPlanReportIsTimeOut(planReportId); + testPlanLog.info("Check PlanReport:" + planReportId + "; result: "+ JSON.toJSONString(checkResult)); + if (checkResult.isTimeOut()) { //判断是否超时。超时时强行停止任务 TestPlanReportExecuteCatch.finishAllTask(planReportId); + checkResult.setFinishedCaseChanged(true); + } + if(checkResult.isFinishedCaseChanged()){ + this.updateExecuteApis(planReportId); } - this.updateExecuteApis(planReportId); } public TestPlanSimpleReportDTO getReport(String reportId) { @@ -1216,19 +1220,61 @@ public class TestPlanReportService { testPlanReportContentMapper.updateByExampleSelective(bloBs,example); } - private boolean checkTestPlanReportIsTimeOut(String planReportId) { + private TestPlanReportExecuteCheckResultDTO checkTestPlanReportIsTimeOut(String planReportId) { + //同步数据库更新状态信息 + try { + this.syncReportStatus(planReportId); + } catch (Exception e) { + LogUtil.info("联动数据库同步执行状态失败! " + e.getMessage()); + LogUtil.error(e); + } TestPlanExecuteInfo executeInfo = TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId); - int unFinishNum = executeInfo.countUnFinishedNum(); - if (unFinishNum > 0) { - //20分钟没有案例执行结果更新，则定位超时 - long lastCountTime = executeInfo.getLastFinishedNumCountTime(); - long nowTime = System.currentTimeMillis(); - testPlanLog.info("ReportId: ["+planReportId+"]; timeCount:"+ (nowTime - lastCountTime)); - if (nowTime - lastCountTime > 1200000) { - return true; + TestPlanReportExecuteCheckResultDTO checkResult = executeInfo.countUnFinishedNum(); + return checkResult; + } + + private void syncReportStatus(String planReportId) { + if (TestPlanReportExecuteCatch.containsReport(planReportId)) { + TestPlanExecuteInfo executeInfo = TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId); + if (executeInfo != null) { + //同步接口案例结果 + Map updateCaseStatusMap = new HashMap<>(); + Map apiCaseReportMap = executeInfo.getRunningApiCaseReportMap(); + if (MapUtils.isNotEmpty(apiCaseReportMap)) { + List execList = extApiDefinitionExecResultMapper.selectStatusByIdList(apiCaseReportMap.keySet()); + for (ApiDefinitionExecResult report : execList) { + String reportId = report.getId(); + String status = report.getStatus(); + if (!StringUtils.equalsAnyIgnoreCase(status, "Running", "Waiting")) { + String planCaseId = apiCaseReportMap.get(reportId); + if (StringUtils.isNotEmpty(planCaseId)) { + updateCaseStatusMap.put(planCaseId, status); + } + } + } + } + //同步场景结果 + Map updateScenarioStatusMap = new HashMap<>(); + Map scenarioReportMap = executeInfo.getRunningScenarioReportMap(); + if (MapUtils.isNotEmpty(scenarioReportMap)) { + List reportList = extApiScenarioReportMapper.selectStatusByIds(scenarioReportMap.keySet()); + for (ApiScenarioReport report : reportList) { + String reportId = report.getId(); + String status = report.getStatus(); + if (!StringUtils.equalsAnyIgnoreCase(status, "Running", "Waiting")) { + String planScenarioId = scenarioReportMap.get(reportId); + if (StringUtils.isNotEmpty(planScenarioId)) { + updateScenarioStatusMap.put(planScenarioId, status); + } + } + } + } + testPlanLog.info("ReportID:"+planReportId+" 本次数据库同步,案例ID："+JSON.toJSONString(apiCaseReportMap.keySet())+";场景ID："+JSON.toJSONString(scenarioReportMap.keySet())+"; 同步结果,案例:"+JSON.toJSONString(updateCaseStatusMap)+";场景："+JSON.toJSONString(updateScenarioStatusMap)); + TestPlanReportExecuteCatch.updateApiTestPlanExecuteInfo(planReportId, updateCaseStatusMap, updateScenarioStatusMap, null); + }else { + testPlanLog.info("同步数据库查询执行信息失败! 报告ID在缓存中未找到！"+planReportId); } } - return false; } private void finishTestPlanReport(String planReportId) { @@ -1239,5 +1285,4 @@ public class TestPlanReportService { } TestPlanReportExecuteCatch.remove(planReportId); } - } diff --git a/backend/src/main/resources/application.properties b/backend/src/main/resources/application.properties index 4757f780f5..3960b21085 100644 --- a/backend/src/main/resources/application.properties +++ b/backend/src/main/resources/application.properties @@ -12,7 +12,7 @@ server.ssl.key-alias=localhost # Hikari spring.datasource.type=com.zaxxer.hikari.HikariDataSource -spring.datasource.hikari.maximum-pool-size=100 +spring.datasource.hikari.maximum-pool-size=200 spring.datasource.hikari.auto-commit=true spring.datasource.hikari.idle-timeout=10000 spring.datasource.hikari.pool-name=DatebookHikariCP @@ -23,7 +23,7 @@ spring.datasource.hikari.connection-test-query=SELECT 1 spring.datasource.quartz.url=${spring.datasource.url} spring.datasource.quartz.username=${spring.datasource.username} spring.datasource.quartz.password=${spring.datasource.password} -spring.datasource.quartz.hikari.maximum-pool-size=50 +spring.datasource.quartz.hikari.maximum-pool-size=200 spring.datasource.quartz.hikari.auto-commit=true spring.datasource.quartz.hikari.idle-timeout=10000 spring.datasource.quartz.hikari.pool-name=DatebookHikariCP @@ -92,7 +92,8 @@ jmeter.home=/opt/jmeter # quartz quartz.enabled=true quartz.scheduler-name=msServerJob -quartz.thread-count=30 +quartz.thread-count=60 +quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock=true # file upload spring.servlet.multipart.max-file-size=500MB spring.servlet.multipart.max-request-size=500MB