From 73577037ff3df2d2048a9b0a2d7ba23e6d49397c Mon Sep 17 00:00:00 2001 From: song-tianyang Date: Mon, 20 Dec 2021 13:55:18 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E6=B5=8B=E8=AF=95=E8=AE=A1=E5=88=92):=20?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=B5=8B=E8=AF=95=E8=AE=A1=E5=88=92=E6=89=A7?= =?UTF-8?q?=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 优化测试计划执行 --- backend/pom.xml | 2 +- .../api/cache/TestPlanExecuteInfo.java | 69 +++++++-- .../api/cache/TestPlanReportExecuteCatch.java | 17 ++- .../api/service/ApiJmeterFileService.java | 2 +- .../api/service/ApiScenarioReportService.java | 22 ++- .../controller/ScheduleController.java | 3 - .../job/sechedule/TestPlanTestJob.java | 26 ++-- .../listener/AppStartListener.java | 28 +++- .../TestPlanReportExecuteCheckResultDTO.java | 11 ++ .../track/service/TestPlanReportService.java | 124 ++++++++++++--- .../track/service/TestPlanService.java | 144 +++++++++--------- .../src/main/resources/application.properties | 10 +- 12 files changed, 328 insertions(+), 130 deletions(-) create mode 100644 backend/src/main/java/io/metersphere/track/dto/TestPlanReportExecuteCheckResultDTO.java diff --git a/backend/pom.xml b/backend/pom.xml index 9b0f02e727..7324424872 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -285,7 +285,7 @@ com.fit2cloud quartz-spring-boot-starter - 0.0.7 + 0.0.9 diff --git a/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java b/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java index d6c3cf79dc..526b0306b6 100644 --- a/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java +++ b/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java @@ -9,6 +9,7 @@ import io.metersphere.base.mapper.ApiScenarioReportMapper; import io.metersphere.commons.constants.TestPlanApiExecuteStatus; import io.metersphere.commons.constants.TestPlanResourceType; import io.metersphere.commons.utils.CommonBeanFactory; +import io.metersphere.track.dto.TestPlanReportExecuteCheckResultDTO; import lombok.Getter; import lombok.Setter; import org.apache.commons.collections.CollectionUtils; @@ -19,6 +20,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * @author song.tianyang @@ -29,13 +31,14 @@ import java.util.Map; public class TestPlanExecuteInfo { private String reportId; private String creator; - private Map apiCaseExecInfo = new HashMap<>(); - private Map apiScenarioCaseExecInfo = new HashMap<>(); - private Map loadCaseExecInfo = new HashMap<>(); + private Map apiCaseExecInfo = new ConcurrentHashMap<>(); + private Map apiScenarioCaseExecInfo = new ConcurrentHashMap<>(); + private Map loadCaseExecInfo = new ConcurrentHashMap<>(); - private Map apiCaseExecuteThreadMap = new HashMap<>(); - private Map apiScenarioThreadMap = new HashMap<>(); - private Map loadCaseReportIdMap = new HashMap<>(); + //案例线程是以reportID为id的。 key:关联表ID value:reportID + private Map apiCaseExecuteThreadMap = new ConcurrentHashMap<>(); + private Map apiScenarioThreadMap = new ConcurrentHashMap<>(); + private Map loadCaseReportIdMap = new ConcurrentHashMap<>(); private Map apiCaseReportMap = new HashMap<>(); private Map apiScenarioReportMap = new HashMap<>(); @@ -82,7 +85,8 @@ public class TestPlanExecuteInfo { } } - public synchronized int countUnFinishedNum() { + public synchronized TestPlanReportExecuteCheckResultDTO countUnFinishedNum() { + TestPlanReportExecuteCheckResultDTO executeCheck = new TestPlanReportExecuteCheckResultDTO(); int unFinishedCount = 0; this.isApiCaseAllExecuted = true; @@ -116,8 +120,21 @@ public class TestPlanExecuteInfo { if (lastUnFinishedNumCount != unFinishedCount) { lastUnFinishedNumCount = unFinishedCount; lastFinishedNumCountTime = System.currentTimeMillis(); + executeCheck.setFinishedCaseChanged(true); + }else if(unFinishedCount == 0){ + executeCheck.setFinishedCaseChanged(true); + }else { + executeCheck.setFinishedCaseChanged(false); } - return unFinishedCount; + executeCheck.setTimeOut(false); + if (unFinishedCount > 0) { + //20分钟没有案例执行结果更新，则定位超时 + long nowTime = System.currentTimeMillis(); + if (nowTime - lastFinishedNumCountTime > 1200000) { + executeCheck.setTimeOut(true); + } + } + return executeCheck; } public Map> getExecutedResult() { @@ -200,13 +217,13 @@ public class TestPlanExecuteInfo { MessageCache.executionQueue.remove(apiScenarioThreadMap.get(resourceId)); } } - if(CollectionUtils.isNotEmpty(updateScenarioReportList)){ + if (CollectionUtils.isNotEmpty(updateScenarioReportList)) { ApiScenarioReportMapper apiScenarioReportMapper = CommonBeanFactory.getBean(ApiScenarioReportMapper.class); ApiScenarioReportExample example = new ApiScenarioReportExample(); example.createCriteria().andIdIn(updateScenarioReportList).andStatusEqualTo("Running"); ApiScenarioReport report = new ApiScenarioReport(); report.setStatus("Error"); - apiScenarioReportMapper.updateByExampleSelective(report,example); + apiScenarioReportMapper.updateByExampleSelective(report, example); } for (Map.Entry entry : loadCaseExecInfo.entrySet()) { @@ -227,7 +244,7 @@ public class TestPlanExecuteInfo { this.countUnFinishedNum(); } - public void updateReport(Map apiCaseExecResultInfo, Map apiScenarioCaseExecResultInfo) { + public synchronized void updateReport(Map apiCaseExecResultInfo, Map apiScenarioCaseExecResultInfo) { if (MapUtils.isNotEmpty(apiCaseExecResultInfo)) { this.apiCaseReportMap.putAll(apiCaseExecResultInfo); } @@ -237,4 +254,34 @@ public class TestPlanExecuteInfo { } } + + public Map getRunningApiCaseReportMap() { + //key: reportId, value: testPlanApiCaseId + Map returnMap = new HashMap<>(); + for (Map.Entry entry : apiCaseExecInfo.entrySet()) { + String planCaseId = entry.getKey(); + String status = entry.getValue(); + if (StringUtils.equalsIgnoreCase(status, TestPlanApiExecuteStatus.RUNNING.name())) { + if (apiCaseExecuteThreadMap.containsKey(planCaseId)) { + returnMap.put(apiCaseExecuteThreadMap.get(planCaseId), planCaseId); + } + } + } + return returnMap; + } + + public Map getRunningScenarioReportMap() { + //key: reportId, value: testPlanApiScenarioId + Map returnMap = new HashMap<>(); + for (Map.Entry entry : apiScenarioCaseExecInfo.entrySet()) { + String planScenarioId = entry.getKey(); + String status = entry.getValue(); + if (StringUtils.equalsIgnoreCase(status, TestPlanApiExecuteStatus.RUNNING.name())) { + if (apiScenarioThreadMap.containsKey(planScenarioId)) { + returnMap.put(apiScenarioThreadMap.get(planScenarioId), planScenarioId); + } + } + } + return returnMap; + } } \ No newline at end of file diff --git a/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java b/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java index d1161c02fb..616c48d7c0 100644 --- a/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java +++ b/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java @@ -1,10 +1,13 @@ package io.metersphere.api.cache; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; /** * @author song.tianyang @@ -48,7 +51,11 @@ public class TestPlanReportExecuteCatch { } public synchronized static boolean containsReport(String reportId){ - return testPlanReportMap != null && testPlanReportMap.containsKey(reportId); + if(StringUtils.isEmpty(reportId)){ + return false; + }else { + return testPlanReportMap != null && testPlanReportMap.containsKey(reportId); + } } public synchronized static void updateApiTestPlanExecuteInfo(String reportId, @@ -97,4 +104,12 @@ public class TestPlanReportExecuteCatch { testPlanReportMap.get(planReportId).finishAllTask(); } } + + public static Set getAllReportId(){ + if (testPlanReportMap != null) { + return testPlanReportMap.keySet(); + }else { + return new HashSet<>(); + } + } } diff --git a/backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java b/backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java index 663b225f15..516a260141 100644 --- a/backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java +++ b/backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java @@ -97,7 +97,7 @@ public class ApiJmeterFileService { } hashTree = apiAutomationService.generateHashTree(item, reportId, planEnvMap); } - return zipFilesToByteArray(remoteTestId, hashTree); + return zipFilesToByteArray(reportId, hashTree); } public byte[] downloadJmx(String runMode, String testId, String reportId, String testPlanScenarioId) { diff --git a/backend/src/main/java/io/metersphere/api/service/ApiScenarioReportService.java b/backend/src/main/java/io/metersphere/api/service/ApiScenarioReportService.java index ff80712ded..87a2ced125 100644 --- a/backend/src/main/java/io/metersphere/api/service/ApiScenarioReportService.java +++ b/backend/src/main/java/io/metersphere/api/service/ApiScenarioReportService.java @@ -161,6 +161,8 @@ public class ApiScenarioReportService { LogUtil.info("从缓存中获取场景报告：【" + test.getName() + "】"); report = MessageCache.scenarioExecResourceLock.get(test.getName()); LogUtil.info("从缓存中获取场景报告：【" + test.getName() + "】是否为空：" + (report == null)); + } else { + LogUtil.info("数据库中获取场景报告结束：" + report.getId()); } if (report != null) { report.setName(report.getScenarioName() + "-" + DateUtils.getTimeStr(System.currentTimeMillis())); @@ -320,6 +322,7 @@ public class ApiScenarioReportService { if (CollectionUtils.isNotEmpty(scenarioResult.getRequestResults())) { startTime = scenarioResult.getRequestResults().get(0).getStartTime(); } + String resultReportId = scenarioResult.getName(); ApiScenarioReport report = editReport(scenarioResult, startTime); if (report != null) { TestResult newResult = createTestResult(result.getTestId(), scenarioResult); @@ -366,7 +369,8 @@ public class ApiScenarioReportService { testPlanApiScenarioMapper.updateByPrimaryKeySelective(testPlanApiScenario); scenarioIdList.add(testPlanApiScenario.getApiScenarioId()); } else { - LogUtil.info("TestPlanReport_Id is null. scenario report id : [" + report.getId() + "]; planScenarioIdArr:[" + report.getScenarioId() + "] DATA:" + JSON.toJSONString(scenarioResult)); + LogUtil.info("Cannot find TestPlanApiScenario!"); + LogUtil.error("TestPlanReport_Id is null. scenario report id : [" + report.getId() + "]; planScenarioIdArr:[" + report.getScenarioId() + "]. plan_scenario_id:" + planScenarioId + ". DATA:" + JSON.toJSONString(scenarioResult)); } report.setEndTime(System.currentTimeMillis()); @@ -376,17 +380,27 @@ public class ApiScenarioReportService { // 报告详情内容 ApiScenarioReportDetail detail = new ApiScenarioReportDetail(); detail.setContent(JSON.toJSONString(newResult).getBytes(StandardCharsets.UTF_8)); - detail.setReportId(report.getId()); + detail.setProjectId(report.getProjectId()); if (StringUtils.isNotEmpty(report.getTriggerMode()) && report.getTriggerMode().equals("CASE")) { report.setTriggerMode(TriggerMode.MANUAL.name()); } + if (StringUtils.equalsIgnoreCase(report.getId(), resultReportId)) { + detail.setReportId(report.getId()); + } else { + detail.setReportId(resultReportId); + LogUtil.info("ReportId" + resultReportId + " has changed!"); + LogUtil.error("ReportId was changed. ScenarioResultData:" + JSON.toJSONString(scenarioResult) + ";\r\n " + + "ApiScenarioReport:" + JSON.toJSONString(report)); + } + try { apiScenarioReportDetailMapper.insert(detail); } catch (Exception e) { - LogUtil.error("存储场景报告出错：" + e.getMessage() + "; 步骤信息:" + JSON.toJSONString(scenarioResult)); + LogUtil.error("Save scenario report error! errorInfo:" + e.getMessage() + "; ScenarioResultData:" + JSON.toJSONString(scenarioResult)); LogUtil.error(e); } + scenarioNames.append(report.getName()).append(","); // 更新场景状态 ApiScenario scenario = apiScenarioMapper.selectByPrimaryKey(report.getScenarioId()); @@ -410,7 +424,7 @@ public class ApiScenarioReportService { MessageCache.executionQueue.remove(report.getId()); reportIds.add(report.getId()); } else { - LogUtil.error("测试计划场景[" + result.getTestId() + "]的场景报告未找到。报告ID:" + scenarioResult.getName() + "。 步骤信息:" + JSON.toJSONString(scenarioResult)); + LogUtil.error("未获取到场景报告。 报告ID：" + scenarioResult.getName() + "。 步骤信息:" + JSON.toJSONString(scenarioResult)); } } testPlanLog.info("TestPlanReportId" + JSONArray.toJSONString(testPlanReportIdList) + " EXECUTE OVER. SCENARIO STATUS : " + JSONObject.toJSONString(scenarioAndErrorMap)); diff --git a/backend/src/main/java/io/metersphere/controller/ScheduleController.java b/backend/src/main/java/io/metersphere/controller/ScheduleController.java index 0b93ea15a1..5dd90cbf22 100644 --- a/backend/src/main/java/io/metersphere/controller/ScheduleController.java +++ b/backend/src/main/java/io/metersphere/controller/ScheduleController.java @@ -2,7 +2,6 @@ package io.metersphere.controller; import com.github.pagehelper.Page; import com.github.pagehelper.PageHelper; -import io.metersphere.api.service.ApiAutomationService; import io.metersphere.base.domain.Schedule; import io.metersphere.controller.request.QueryScheduleRequest; import io.metersphere.controller.request.ScheduleRequest; @@ -18,8 +17,6 @@ import java.util.List; public class ScheduleController { @Resource private ScheduleService scheduleService; - @Resource - private ApiAutomationService apiAutomationService; @PostMapping("/list/{goPage}/{pageSize}") public List list(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody QueryScheduleRequest request) { diff --git a/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java b/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java index 1f03186e9c..bcb3ddc745 100644 --- a/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java +++ b/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java @@ -3,6 +3,7 @@ package io.metersphere.job.sechedule; import io.metersphere.commons.constants.ReportTriggerMode; import io.metersphere.commons.constants.ScheduleGroup; import io.metersphere.commons.utils.CommonBeanFactory; +import io.metersphere.commons.utils.LogUtil; import io.metersphere.track.service.TestPlanService; import org.quartz.*; @@ -16,22 +17,9 @@ import org.quartz.*; public class TestPlanTestJob extends MsScheduleJob { private String projectID; - - // private PerformanceTestService performanceTestService; -// private TestPlanScenarioCaseService testPlanScenarioCaseService; -// private TestPlanApiCaseService testPlanApiCaseService; -// private ApiTestCaseService apiTestCaseService; -// private TestPlanReportService testPlanReportService; -// private TestPlanLoadCaseService testPlanLoadCaseService; private TestPlanService testPlanService; public TestPlanTestJob() { -// this.performanceTestService = CommonBeanFactory.getBean(PerformanceTestService.class); -// this.testPlanScenarioCaseService = CommonBeanFactory.getBean(TestPlanScenarioCaseService.class); -// this.testPlanApiCaseService = CommonBeanFactory.getBean(TestPlanApiCaseService.class); -// this.apiTestCaseService = CommonBeanFactory.getBean(ApiTestCaseService.class); -// this.testPlanReportService = CommonBeanFactory.getBean(TestPlanReportService.class); -// this.testPlanLoadCaseService = CommonBeanFactory.getBean(TestPlanLoadCaseService.class); this.testPlanService = CommonBeanFactory.getBean(TestPlanService.class); @@ -63,7 +51,17 @@ public class TestPlanTestJob extends MsScheduleJob { JobDataMap jobDataMap = context.getJobDetail().getJobDataMap(); String config = jobDataMap.getString("config"); - testPlanService.run(this.resourceId, this.projectID, this.userId, ReportTriggerMode.SCHEDULE.name(),config); + String runResourceId = this.resourceId; + String runProjectId = this.projectID; + String runUserId = this.userId; + LogUtil.info("Start test_plan_scehdule. test_plan_id:" + runProjectId); + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + testPlanService.run(runResourceId, runProjectId, runUserId, ReportTriggerMode.SCHEDULE.name(),config); + } + }); + thread.start(); } public static JobKey getJobKey(String testId) { diff --git a/backend/src/main/java/io/metersphere/listener/AppStartListener.java b/backend/src/main/java/io/metersphere/listener/AppStartListener.java index f7a9414d58..73ea21a23d 100644 --- a/backend/src/main/java/io/metersphere/listener/AppStartListener.java +++ b/backend/src/main/java/io/metersphere/listener/AppStartListener.java @@ -66,9 +66,23 @@ public class AppStartListener implements ApplicationListener 0){ + planListenerExecutorService.setMaximumPoolSize(threadCount); + } + } + public List list(QueryTestPlanReportRequest request) { List list = new ArrayList<>(); request.setOrders(ServiceUtils.getDefaultOrder(request.getOrders())); @@ -551,8 +563,6 @@ public class TestPlanReportService { boolean scenarioIsOk = executeInfo.isScenarioAllExecuted(); boolean performanceIsOk = executeInfo.isLoadCaseAllExecuted(); - testPlanLog.info("ReportId[" + testPlanReport.getId() + "] count over. Testplan Execute Result: Api is over ->" + apiCaseIsOk + "; scenario is over ->" + scenarioIsOk + "; performance is over ->" + performanceIsOk); - if (apiCaseIsOk) { testPlanReport.setIsApiCaseExecuting(false); } @@ -926,7 +936,7 @@ public class TestPlanReportService { testPlanLog.info("TestPlanReportId[" + testPlanReport.getId() + "] SELECT performance BATCH OVER:" + JSONArray.toJSONString(selectList)); if (performaneReportIDList.isEmpty()) { testPlanLog.info("TestPlanReportId[" + testPlanReport.getId() + "] performance EXECUTE OVER. TRIGGER_MODE:" + triggerMode + ",REsult:" + JSONObject.toJSONString(finishLoadTestId)); - if (StringUtils.equalsAnyIgnoreCase(triggerMode, ReportTriggerMode.API.name() ,ReportTriggerMode.MANUAL.name())) { + if (StringUtils.equalsAnyIgnoreCase(triggerMode, ReportTriggerMode.API.name(), ReportTriggerMode.MANUAL.name())) { for (String string : finishLoadTestId.keySet()) { String reportId = caseReportMap.get(string); TestPlanLoadCaseWithBLOBs updateDTO = new TestPlanLoadCaseWithBLOBs(); @@ -1075,12 +1085,16 @@ public class TestPlanReportService { } public void countReport(String planReportId) { - boolean isTimeOut = this.checkTestPlanReportIsTimeOut(planReportId); - if (isTimeOut) { + TestPlanReportExecuteCheckResultDTO checkResult = this.checkTestPlanReportIsTimeOut(planReportId); + testPlanLog.info("Check PlanReport:" + planReportId + "; result: "+ JSON.toJSONString(checkResult)); + if (checkResult.isTimeOut()) { //判断是否超时。超时时强行停止任务 TestPlanReportExecuteCatch.finishAllTask(planReportId); + checkResult.setFinishedCaseChanged(true); + } + if(checkResult.isFinishedCaseChanged()){ + this.updateExecuteApis(planReportId); } - this.updateExecuteApis(planReportId); } public TestPlanSimpleReportDTO getReport(String reportId) { @@ -1188,7 +1202,7 @@ public class TestPlanReportService { if (StringUtils.isNotBlank(testPlanReportContent.getLoadAllCases())) { List allCases = JSONObject.parseArray(testPlanReportContent.getLoadAllCases(), TestPlanLoadCaseDTO.class); - if(!allCases.isEmpty()){ + if (!allCases.isEmpty()) { isTaskRunning = true; } } @@ -1213,22 +1227,68 @@ public class TestPlanReportService { bloBs.setEndTime(endTime); TestPlanReportContentExample example = new TestPlanReportContentExample(); example.createCriteria().andTestPlanReportIdEqualTo(testPlanReport.getId()); - testPlanReportContentMapper.updateByExampleSelective(bloBs,example); + testPlanReportContentMapper.updateByExampleSelective(bloBs, example); } - private boolean checkTestPlanReportIsTimeOut(String planReportId) { - TestPlanExecuteInfo executeInfo = TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId); - int unFinishNum = executeInfo.countUnFinishedNum(); - if (unFinishNum > 0) { - //20分钟没有案例执行结果更新，则定位超时 - long lastCountTime = executeInfo.getLastFinishedNumCountTime(); - long nowTime = System.currentTimeMillis(); - testPlanLog.info("ReportId: ["+planReportId+"]; timeCount:"+ (nowTime - lastCountTime)); - if (nowTime - lastCountTime > 1200000) { - return true; - } + private TestPlanReportExecuteCheckResultDTO checkTestPlanReportIsTimeOut(String planReportId) { + //同步数据库更新状态信息 + try { + this.syncReportStatus(planReportId); + } catch (Exception e) { + LogUtil.info("联动数据库同步执行状态失败! " + e.getMessage()); + LogUtil.error(e); } - return false; + + TestPlanExecuteInfo executeInfo = TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId); + TestPlanReportExecuteCheckResultDTO checkResult = executeInfo.countUnFinishedNum(); + + return checkResult; + } + + private void syncReportStatus(String planReportId) { + if (TestPlanReportExecuteCatch.containsReport(planReportId)) { + TestPlanExecuteInfo executeInfo = TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId); + if (executeInfo != null) { + //同步接口案例结果 + Map updateCaseStatusMap = new HashMap<>(); + Map apiCaseReportMap = executeInfo.getRunningApiCaseReportMap(); + if (MapUtils.isNotEmpty(apiCaseReportMap)) { + List execList = extApiDefinitionExecResultMapper.selectStatusByIdList(apiCaseReportMap.keySet()); + for (ApiDefinitionExecResult report : execList) { + String reportId = report.getId(); + String status = report.getStatus(); + if (!StringUtils.equalsAnyIgnoreCase(status, "Running", "Waiting")) { + String planCaseId = apiCaseReportMap.get(reportId); + if (StringUtils.isNotEmpty(planCaseId)) { + updateCaseStatusMap.put(planCaseId, status); + } + } + } + } + //同步场景结果 + Map updateScenarioStatusMap = new HashMap<>(); + Map scenarioReportMap = executeInfo.getRunningScenarioReportMap(); + if (MapUtils.isNotEmpty(scenarioReportMap)) { + List reportList = extApiScenarioReportMapper.selectStatusByIds(scenarioReportMap.keySet()); + for (ApiScenarioReport report : reportList) { + String reportId = report.getId(); + String status = report.getStatus(); + if (!StringUtils.equalsAnyIgnoreCase(status, "Running", "Waiting")) { + String planScenarioId = scenarioReportMap.get(reportId); + if (StringUtils.isNotEmpty(planScenarioId)) { + updateScenarioStatusMap.put(planScenarioId, status); + } + } + } + } + testPlanLog.info("ReportID:"+planReportId+" 本次数据库同步,案例ID："+JSON.toJSONString(apiCaseReportMap.keySet())+";场景ID："+JSON.toJSONString(scenarioReportMap.keySet())+"; 同步结果,案例:"+JSON.toJSONString(updateCaseStatusMap)+";场景："+JSON.toJSONString(updateScenarioStatusMap)); + TestPlanReportExecuteCatch.updateApiTestPlanExecuteInfo(planReportId, updateCaseStatusMap, updateScenarioStatusMap, null); + }else { + testPlanLog.info("同步数据库查询执行信息失败! 报告ID在缓存中未找到！"+planReportId); + } + + } + } private void finishTestPlanReport(String planReportId) { @@ -1240,4 +1300,26 @@ public class TestPlanReportService { TestPlanReportExecuteCatch.remove(planReportId); } + public void startTestPlanExecuteListen(String reportId){ + planListenerExecutorService.submit(() -> { + while (TestPlanReportExecuteCatch.containsReport(reportId)){ + //检查是否存在数据库里 + TestPlanReportExample example = new TestPlanReportExample(); + example.createCriteria().andIdEqualTo(reportId); + long count = testPlanReportMapper.countByExample(example); + if(count > 0){ + testPlanLog.info("Start check testPlanReport:" + reportId); + countReport(reportId); + try { + Thread.sleep(10000); + }catch (Exception e){ + LogUtil.info(e); + } + }else { + TestPlanReportExecuteCatch.remove(reportId); + } + } + }); + } + } diff --git a/backend/src/main/java/io/metersphere/track/service/TestPlanService.java b/backend/src/main/java/io/metersphere/track/service/TestPlanService.java index 37bd04d9c4..23fcb379a6 100644 --- a/backend/src/main/java/io/metersphere/track/service/TestPlanService.java +++ b/backend/src/main/java/io/metersphere/track/service/TestPlanService.java @@ -85,8 +85,8 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.*; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -197,7 +197,13 @@ public class TestPlanService { @Resource private LoadTestMapper loadTestMapper; - private final ExecutorService executorService = Executors.newFixedThreadPool(40, new NamedThreadFactory("TestPlanService")); + private final ThreadPoolExecutor executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(20, new NamedThreadFactory("TestPlanService")); + + public void resetThreadPool(int threadCount){ + if(threadCount > 0){ + executorService.setMaximumPoolSize(threadCount); + } + } public synchronized TestPlan addTestPlan(AddTestPlanRequest testPlan) { if (getTestPlanByName(testPlan.getName()).size() > 0) { @@ -1120,7 +1126,6 @@ public class TestPlanService { TestPlanReport testPlanReport = reportInfoDTO.getTestPlanReport(); Map planScenarioIdsMap = reportInfoDTO.getPlanScenarioIdMap(); Map planApiCaseMap = reportInfoDTO.getApiTestCaseDataMap(); - Map performanceIdMap = reportInfoDTO.getPerformanceIdMap(); if (runModeConfig.getMode().equals(RunModeConstants.PARALLEL.toString())) { // 校验并发数量 @@ -1141,8 +1146,22 @@ public class TestPlanService { extTestPlanMapper.updateActualEndTimeIsNullById(testPlanID); String planReportId = testPlanReport.getId(); testPlanLog.info("ReportId[" + planReportId + "] created. TestPlanID:[" + testPlanID + "]. " + "API Run Config:【" + apiRunConfig + "】"); - //开启测试计划执行状态的监听 - this.listenTaskExecuteStatus(planReportId); + + RunModeConfig finalRunModeConfig = runModeConfig; + executorService.submit(() -> { + testPlanLog.info("ReportId[" + planReportId + "] start execute."); + this.executeTestPlan(reportInfoDTO, triggerMode, projectID, userId, finalRunModeConfig); + testPlanLog.info("ReportId[" + planReportId + "] is executing."); + }); + return testPlanReport.getId(); + } + + private void executeTestPlan(TestPlanScheduleReportInfoDTO reportInfoDTO, String triggerMode, String projectID, String userId, RunModeConfig runModeConfig) { + TestPlanReport testPlanReport = reportInfoDTO.getTestPlanReport(); + String planReportId = testPlanReport.getId(); + Map planScenarioIdsMap = reportInfoDTO.getPlanScenarioIdMap(); + Map planApiCaseMap = reportInfoDTO.getApiTestCaseDataMap(); + Map performanceIdMap = reportInfoDTO.getPerformanceIdMap(); //不同任务的执行ID Map executePerformanceIdMap = new HashMap<>(); @@ -1208,80 +1227,66 @@ public class TestPlanService { for (String id : planScenarioIdsMap.keySet()) { executeScenarioCaseIdMap.put(id, TestPlanApiExecuteStatus.RUNNING.name()); } - testPlanLog.info("ReportId[" + planReportId + "] start run. TestPlanID:[" + testPlanID + "]. Execute api :" + JSONObject.toJSONString(executeApiCaseIdMap) + "; Execute scenario:" + JSONObject.toJSONString(executeScenarioCaseIdMap) + "; Execute performance:" + JSONObject.toJSONString(executePerformanceIdMap)); + testPlanLog.info("ReportId[" + planReportId + "] start run. TestPlanID:[" + testPlanReport.getTestPlanId() + "]. Execute api :" + JSONObject.toJSONString(executeApiCaseIdMap) + "; Execute scenario:" + JSONObject.toJSONString(executeScenarioCaseIdMap) + "; Execute performance:" + JSONObject.toJSONString(executePerformanceIdMap)); TestPlanReportExecuteCatch.updateApiTestPlanExecuteInfo(planReportId, executeApiCaseIdMap, executeScenarioCaseIdMap, executePerformanceIdMap); //执行接口案例任务 - this.executeApiTestCase(triggerMode, planReportId, userId, new ArrayList<>(planApiCaseMap.keySet()), runModeConfig); + if(!planApiCaseMap.isEmpty()){ + this.executeApiTestCase(triggerMode, planReportId, userId, new ArrayList<>(planApiCaseMap.keySet()), runModeConfig); + } //执行场景执行任务 - this.executeScenarioCase(planReportId, testPlanID, projectID, runModeConfig, triggerMode, userId, planScenarioIdsMap); - return testPlanReport.getId(); - } - - private void listenTaskExecuteStatus(String planReportId) { - executorService.submit(() -> { - try { - //10s 查询一次状态 - Thread.sleep(10000); - while (TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId) != null) { - testPlanReportService.countReport(planReportId); - Thread.sleep(10000); - } - } catch (InterruptedException e) { - TestPlanReportExecuteCatch.remove(planReportId); - LogUtil.error(e); - } - }); + if(!planScenarioIdsMap.isEmpty()){ + this.executeScenarioCase(planReportId, testPlanReport.getTestPlanId(), projectID, runModeConfig, triggerMode, userId, planScenarioIdsMap); + } + testPlanReportService.startTestPlanExecuteListen(planReportId); } private void executeApiTestCase(String triggerMode, String planReportId, String userId, List planCaseIds, RunModeConfig runModeConfig) { - executorService.submit(() -> { - BatchRunDefinitionRequest request = new BatchRunDefinitionRequest(); - if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) { - request.setTriggerMode(ApiRunMode.JENKINS_API_PLAN.name()); - } else if (StringUtils.equals(triggerMode, ReportTriggerMode.MANUAL.name())) { - request.setTriggerMode(ApiRunMode.MANUAL_PLAN.name()); - } else { - request.setTriggerMode(ApiRunMode.SCHEDULE_API_PLAN.name()); - } - request.setPlanIds(planCaseIds); - request.setPlanReportId(planReportId); - request.setConfig(runModeConfig); - request.setUserId(userId); - testPlanApiCaseService.run(request); - }); + + BatchRunDefinitionRequest request = new BatchRunDefinitionRequest(); + if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) { + request.setTriggerMode(ApiRunMode.JENKINS_API_PLAN.name()); + } else if (StringUtils.equals(triggerMode, ReportTriggerMode.MANUAL.name())) { + request.setTriggerMode(ApiRunMode.MANUAL_PLAN.name()); + } else { + request.setTriggerMode(ApiRunMode.SCHEDULE_API_PLAN.name()); + } + + request.setPlanIds(planCaseIds); + request.setPlanReportId(planReportId); + request.setConfig(runModeConfig); + request.setUserId(userId); + testPlanApiCaseService.run(request); } private void executeScenarioCase(String planReportId, String testPlanID, String projectID, RunModeConfig runModeConfig, String triggerMode, String userId, Map planScenarioIdMap) { - executorService.submit(() -> { - if (!planScenarioIdMap.isEmpty()) { - SchedulePlanScenarioExecuteRequest scenarioRequest = new SchedulePlanScenarioExecuteRequest(); - String senarionReportID = UUID.randomUUID().toString(); - scenarioRequest.setId(senarionReportID); - scenarioRequest.setReportId(senarionReportID); - scenarioRequest.setProjectId(projectID); - if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) { - scenarioRequest.setTriggerMode(ReportTriggerMode.API.name()); - scenarioRequest.setRunMode(ApiRunMode.JENKINS_SCENARIO_PLAN.name()); - } else if (StringUtils.equals(triggerMode, ReportTriggerMode.MANUAL.name())) { - scenarioRequest.setTriggerMode(ReportTriggerMode.MANUAL.name()); - scenarioRequest.setRunMode(ApiRunMode.JENKINS_SCENARIO_PLAN.name()); - } else { - scenarioRequest.setTriggerMode(ReportTriggerMode.SCHEDULE.name()); - scenarioRequest.setRunMode(ApiRunMode.SCHEDULE_SCENARIO_PLAN.name()); - } - scenarioRequest.setExecuteType(ExecuteType.Saved.name()); - Map> testPlanScenarioIdMap = new HashMap<>(); - testPlanScenarioIdMap.put(testPlanID, planScenarioIdMap); - scenarioRequest.setTestPlanScenarioIDMap(testPlanScenarioIdMap); - scenarioRequest.setReportUserID(userId); - scenarioRequest.setTestPlanID(testPlanID); - - scenarioRequest.setTestPlanReportId(planReportId); - - scenarioRequest.setConfig(runModeConfig); - this.scenarioRunModeConfig(scenarioRequest); + if (!planScenarioIdMap.isEmpty()) { + SchedulePlanScenarioExecuteRequest scenarioRequest = new SchedulePlanScenarioExecuteRequest(); + String senarionReportID = UUID.randomUUID().toString(); + scenarioRequest.setId(senarionReportID); + scenarioRequest.setReportId(senarionReportID); + scenarioRequest.setProjectId(projectID); + if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) { + scenarioRequest.setTriggerMode(ReportTriggerMode.API.name()); + scenarioRequest.setRunMode(ApiRunMode.JENKINS_SCENARIO_PLAN.name()); + } else if (StringUtils.equals(triggerMode, ReportTriggerMode.MANUAL.name())) { + scenarioRequest.setTriggerMode(ReportTriggerMode.MANUAL.name()); + scenarioRequest.setRunMode(ApiRunMode.JENKINS_SCENARIO_PLAN.name()); + } else { + scenarioRequest.setTriggerMode(ReportTriggerMode.SCHEDULE.name()); + scenarioRequest.setRunMode(ApiRunMode.SCHEDULE_SCENARIO_PLAN.name()); } - }); + scenarioRequest.setExecuteType(ExecuteType.Saved.name()); + Map> testPlanScenarioIdMap = new HashMap<>(); + testPlanScenarioIdMap.put(testPlanID, planScenarioIdMap); + scenarioRequest.setTestPlanScenarioIDMap(testPlanScenarioIdMap); + scenarioRequest.setReportUserID(userId); + scenarioRequest.setTestPlanID(testPlanID); + + scenarioRequest.setTestPlanReportId(planReportId); + + scenarioRequest.setConfig(runModeConfig); + this.scenarioRunModeConfig(scenarioRequest); + } } public String getLogDetails(String id) { @@ -1822,7 +1827,6 @@ public class TestPlanService { config = JSONObject.parseObject(reportConfig); } TestPlanSimpleReportDTO report = getReport(planId); - buildFunctionalReport(report, config, planId); buildApiReport(report, config, executeInfo, isFinish); buildLoadReport(report, config, executeInfo, planId, false); return report; diff --git a/backend/src/main/resources/application.properties b/backend/src/main/resources/application.properties index 4757f780f5..2a6486561a 100644 --- a/backend/src/main/resources/application.properties +++ b/backend/src/main/resources/application.properties @@ -12,7 +12,7 @@ server.ssl.key-alias=localhost # Hikari spring.datasource.type=com.zaxxer.hikari.HikariDataSource -spring.datasource.hikari.maximum-pool-size=100 +spring.datasource.hikari.maximum-pool-size=200 spring.datasource.hikari.auto-commit=true spring.datasource.hikari.idle-timeout=10000 spring.datasource.hikari.pool-name=DatebookHikariCP @@ -23,7 +23,7 @@ spring.datasource.hikari.connection-test-query=SELECT 1 spring.datasource.quartz.url=${spring.datasource.url} spring.datasource.quartz.username=${spring.datasource.username} spring.datasource.quartz.password=${spring.datasource.password} -spring.datasource.quartz.hikari.maximum-pool-size=50 +spring.datasource.quartz.hikari.maximum-pool-size=200 spring.datasource.quartz.hikari.auto-commit=true spring.datasource.quartz.hikari.idle-timeout=10000 spring.datasource.quartz.hikari.pool-name=DatebookHikariCP @@ -92,7 +92,10 @@ jmeter.home=/opt/jmeter # quartz quartz.enabled=true quartz.scheduler-name=msServerJob -quartz.thread-count=30 +quartz.thread-count=60 +quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock=true +#schedule +testplan.thread-count=40 # file upload spring.servlet.multipart.max-file-size=500MB spring.servlet.multipart.max-request-size=500MB @@ -103,3 +106,4 @@ management.endpoints.web.exposure.include=* +