From b7cd21bc4a469943adf9d6a8affc835671c2b7d9 Mon Sep 17 00:00:00 2001 From: song-tianyang Date: Fri, 17 Dec 2021 16:06:58 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E6=B5=8B=E8=AF=95=E8=AE=A1=E5=88=92):=20?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=B5=8B=E8=AF=95=E8=AE=A1=E5=88=92=E6=89=A7?= =?UTF-8?q?=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 优化测试计划执行，更改配置信息 --- .../api/cache/TestPlanExecuteInfo.java | 19 ++- .../api/cache/TestPlanReportExecuteCatch.java | 22 ++- .../api/service/ApiJmeterFileService.java | 2 +- .../job/sechedule/TestPlanTestJob.java | 4 +- .../listener/AppStartListener.java | 15 +- .../TestPlanReportExecuteCheckResultDTO.java | 11 ++ .../track/service/TestPlanReportService.java | 59 +++++--- .../track/service/TestPlanService.java | 143 +++++++++--------- .../src/main/resources/application.properties | 7 +- 9 files changed, 184 insertions(+), 98 deletions(-) create mode 100644 backend/src/main/java/io/metersphere/track/dto/TestPlanReportExecuteCheckResultDTO.java diff --git a/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java b/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java index 0c3c30ca9e..526b0306b6 100644 --- a/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java +++ b/backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java @@ -9,6 +9,7 @@ import io.metersphere.base.mapper.ApiScenarioReportMapper; import io.metersphere.commons.constants.TestPlanApiExecuteStatus; import io.metersphere.commons.constants.TestPlanResourceType; import io.metersphere.commons.utils.CommonBeanFactory; +import io.metersphere.track.dto.TestPlanReportExecuteCheckResultDTO; import lombok.Getter; import lombok.Setter; import org.apache.commons.collections.CollectionUtils; @@ -84,7 +85,8 @@ public class TestPlanExecuteInfo { } } - public synchronized int countUnFinishedNum() { + public synchronized TestPlanReportExecuteCheckResultDTO countUnFinishedNum() { + TestPlanReportExecuteCheckResultDTO executeCheck = new TestPlanReportExecuteCheckResultDTO(); int unFinishedCount = 0; this.isApiCaseAllExecuted = true; @@ -118,8 +120,21 @@ public class TestPlanExecuteInfo { if (lastUnFinishedNumCount != unFinishedCount) { lastUnFinishedNumCount = unFinishedCount; lastFinishedNumCountTime = System.currentTimeMillis(); + executeCheck.setFinishedCaseChanged(true); + }else if(unFinishedCount == 0){ + executeCheck.setFinishedCaseChanged(true); + }else { + executeCheck.setFinishedCaseChanged(false); } - return unFinishedCount; + executeCheck.setTimeOut(false); + if (unFinishedCount > 0) { + //20分钟没有案例执行结果更新，则定位超时 + long nowTime = System.currentTimeMillis(); + if (nowTime - lastFinishedNumCountTime > 1200000) { + executeCheck.setTimeOut(true); + } + } + return executeCheck; } public Map> getExecutedResult() { diff --git a/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java b/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java index 518556b70f..d52fc21a99 100644 --- a/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java +++ b/backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java @@ -1,14 +1,16 @@ package io.metersphere.api.cache; -import java.util.HashMap; -import java.util.Map; +import org.apache.commons.lang3.StringUtils; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; /** * @author song.tianyang * @Date 2021/8/20 3:29 下午 */ public class TestPlanReportExecuteCatch { - private static Map testPlanReportMap = new HashMap<>(); + private static Map testPlanReportMap = new ConcurrentHashMap<>(); private TestPlanReportExecuteCatch() { } @@ -44,7 +46,11 @@ public class TestPlanReportExecuteCatch { } public synchronized static boolean containsReport(String reportId){ - return testPlanReportMap != null && testPlanReportMap.containsKey(reportId); + if(StringUtils.isEmpty(reportId)){ + return false; + }else { + return testPlanReportMap != null && testPlanReportMap.containsKey(reportId); + } } public synchronized static void updateApiTestPlanExecuteInfo(String reportId, @@ -92,4 +98,12 @@ public class TestPlanReportExecuteCatch { testPlanReportMap.get(planReportId).finishAllTask(); } } + + public static Set getAllReportId(){ + if (testPlanReportMap != null) { + return testPlanReportMap.keySet(); + }else { + return new HashSet<>(); + } + } } diff --git a/backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java b/backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java index 32303edcc2..15b4fadea1 100644 --- a/backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java +++ b/backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java @@ -82,7 +82,7 @@ public class ApiJmeterFileService { } hashTree = apiAutomationService.generateHashTree(item, reportId, planEnvMap); } - return zipFilesToByteArray(remoteTestId, hashTree); + return zipFilesToByteArray(reportId, hashTree); } public byte[] downloadJmx(String runMode, String testId, String reportId, String testPlanScenarioId) { diff --git a/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java b/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java index a30ea06e0e..bcb3ddc745 100644 --- a/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java +++ b/backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java @@ -3,6 +3,7 @@ package io.metersphere.job.sechedule; import io.metersphere.commons.constants.ReportTriggerMode; import io.metersphere.commons.constants.ScheduleGroup; import io.metersphere.commons.utils.CommonBeanFactory; +import io.metersphere.commons.utils.LogUtil; import io.metersphere.track.service.TestPlanService; import org.quartz.*; @@ -53,13 +54,14 @@ public class TestPlanTestJob extends MsScheduleJob { String runResourceId = this.resourceId; String runProjectId = this.projectID; String runUserId = this.userId; + LogUtil.info("Start test_plan_scehdule. test_plan_id:" + runProjectId); Thread thread = new Thread(new Runnable() { @Override public void run() { testPlanService.run(runResourceId, runProjectId, runUserId, ReportTriggerMode.SCHEDULE.name(),config); } }); - thread.run(); + thread.start(); } public static JobKey getJobKey(String testId) { diff --git a/backend/src/main/java/io/metersphere/listener/AppStartListener.java b/backend/src/main/java/io/metersphere/listener/AppStartListener.java index 55f334b6f9..73ea21a23d 100644 --- a/backend/src/main/java/io/metersphere/listener/AppStartListener.java +++ b/backend/src/main/java/io/metersphere/listener/AppStartListener.java @@ -66,6 +66,10 @@ public class AppStartListener implements ApplicationListener 0){ + planListenerExecutorService.setMaximumPoolSize(threadCount); + } + } + public List list(QueryTestPlanReportRequest request) { List list = new ArrayList<>(); request.setOrders(ServiceUtils.getDefaultOrder(request.getOrders())); @@ -584,8 +592,6 @@ public class TestPlanReportService { boolean scenarioIsOk = executeInfo.isScenarioAllExecuted(); boolean performanceIsOk = executeInfo.isLoadCaseAllExecuted(); - testPlanLog.info("ReportId[" + testPlanReport.getId() + "] count over. Testplan Execute Result: Api is over ->" + apiCaseIsOk + "; scenario is over ->" + scenarioIsOk + "; performance is over ->" + performanceIsOk); - if (apiCaseIsOk) { testPlanReport.setIsApiCaseExecuting(false); } @@ -1113,12 +1119,16 @@ public class TestPlanReportService { } public void countReport(String planReportId) { - boolean isTimeOut = this.checkTestPlanReportIsTimeOut(planReportId); - if (isTimeOut) { + TestPlanReportExecuteCheckResultDTO checkResult = this.checkTestPlanReportIsTimeOut(planReportId); + testPlanLog.info("Check PlanReport:" + planReportId + "; result: "+ JSON.toJSONString(checkResult)); + if (checkResult.isTimeOut()) { //判断是否超时。超时时强行停止任务 TestPlanReportExecuteCatch.finishAllTask(planReportId); + checkResult.setFinishedCaseChanged(true); + } + if(checkResult.isFinishedCaseChanged()){ + this.updateExecuteApis(planReportId); } - this.updateExecuteApis(planReportId); } public TestPlanSimpleReportDTO getReport(String reportId) { @@ -1254,7 +1264,7 @@ public class TestPlanReportService { testPlanReportContentMapper.updateByExampleSelective(bloBs, example); } - private boolean checkTestPlanReportIsTimeOut(String planReportId) { + private TestPlanReportExecuteCheckResultDTO checkTestPlanReportIsTimeOut(String planReportId) { //同步数据库更新状态信息 try { this.syncReportStatus(planReportId); @@ -1264,16 +1274,9 @@ public class TestPlanReportService { } TestPlanExecuteInfo executeInfo = TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId); - int unFinishNum = executeInfo.countUnFinishedNum(); - if (unFinishNum > 0) { - //20分钟没有案例执行结果更新，则定位超时 - long lastCountTime = executeInfo.getLastFinishedNumCountTime(); - long nowTime = System.currentTimeMillis(); - if (nowTime - lastCountTime > 1200000) { - return true; - } - } - return false; + TestPlanReportExecuteCheckResultDTO checkResult = executeInfo.countUnFinishedNum(); + + return checkResult; } private void syncReportStatus(String planReportId) { @@ -1331,4 +1334,26 @@ public class TestPlanReportService { TestPlanReportExecuteCatch.remove(planReportId); } + public void startTestPlanExecuteListen(String reportId){ + planListenerExecutorService.submit(() -> { + while (TestPlanReportExecuteCatch.containsReport(reportId)){ + //检查是否存在数据库里 + TestPlanReportExample example = new TestPlanReportExample(); + example.createCriteria().andIdEqualTo(reportId); + long count = testPlanReportMapper.countByExample(example); + if(count > 0){ + testPlanLog.info("Start check testPlanReport:" + reportId); + countReport(reportId); + try { + Thread.sleep(10000); + }catch (Exception e){ + LogUtil.info(e); + } + }else { + TestPlanReportExecuteCatch.remove(reportId); + } + } + }); + } + } diff --git a/backend/src/main/java/io/metersphere/track/service/TestPlanService.java b/backend/src/main/java/io/metersphere/track/service/TestPlanService.java index cdbfca9a1c..1426075819 100644 --- a/backend/src/main/java/io/metersphere/track/service/TestPlanService.java +++ b/backend/src/main/java/io/metersphere/track/service/TestPlanService.java @@ -70,7 +70,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; @@ -84,8 +83,8 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.*; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -190,7 +189,13 @@ public class TestPlanService { @Resource private TestPlanFollowMapper testPlanFollowMapper; - private final ExecutorService executorService = Executors.newFixedThreadPool(40, new NamedThreadFactory("TestPlanService")); + private final ThreadPoolExecutor executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(20, new NamedThreadFactory("TestPlanService")); + + public void resetThreadPool(int threadCount){ + if(threadCount > 0){ + executorService.setMaximumPoolSize(threadCount); + } + } public synchronized TestPlan addTestPlan(AddTestPlanRequest testPlan) { if (getTestPlanByName(testPlan.getName()).size() > 0) { @@ -1095,7 +1100,6 @@ public class TestPlanService { TestPlanReport testPlanReport = reportInfoDTO.getTestPlanReport(); Map planScenarioIdsMap = reportInfoDTO.getPlanScenarioIdMap(); Map planApiCaseMap = reportInfoDTO.getApiTestCaseDataMap(); - Map performanceIdMap = reportInfoDTO.getPerformanceIdMap(); if (runModeConfig.getMode().equals(RunModeConstants.PARALLEL.toString())) { // 校验并发数量 @@ -1116,8 +1120,22 @@ public class TestPlanService { extTestPlanMapper.updateActualEndTimeIsNullById(testPlanID); String planReportId = testPlanReport.getId(); testPlanLog.info("ReportId[" + planReportId + "] created. TestPlanID:[" + testPlanID + "]. " + "API Run Config:【" + apiRunConfig + "】"); - //开启测试计划执行状态的监听 - this.listenTaskExecuteStatus(planReportId); + + RunModeConfig finalRunModeConfig = runModeConfig; + executorService.submit(() -> { + testPlanLog.info("ReportId[" + planReportId + "] start execute."); + this.executeTestPlan(reportInfoDTO, triggerMode, projectID, userId, finalRunModeConfig); + testPlanLog.info("ReportId[" + planReportId + "] is executing."); + }); + return testPlanReport.getId(); + } + + private void executeTestPlan(TestPlanScheduleReportInfoDTO reportInfoDTO, String triggerMode, String projectID, String userId, RunModeConfig runModeConfig) { + TestPlanReport testPlanReport = reportInfoDTO.getTestPlanReport(); + String planReportId = testPlanReport.getId(); + Map planScenarioIdsMap = reportInfoDTO.getPlanScenarioIdMap(); + Map planApiCaseMap = reportInfoDTO.getApiTestCaseDataMap(); + Map performanceIdMap = reportInfoDTO.getPerformanceIdMap(); //不同任务的执行ID Map executePerformanceIdMap = new HashMap<>(); @@ -1180,80 +1198,66 @@ public class TestPlanService { for (String id : planScenarioIdsMap.keySet()) { executeScenarioCaseIdMap.put(id, TestPlanApiExecuteStatus.RUNNING.name()); } - testPlanLog.info("ReportId[" + planReportId + "] start run. TestPlanID:[" + testPlanID + "]. Execute api :" + JSONObject.toJSONString(executeApiCaseIdMap) + "; Execute scenario:" + JSONObject.toJSONString(executeScenarioCaseIdMap) + "; Execute performance:" + JSONObject.toJSONString(executePerformanceIdMap)); + testPlanLog.info("ReportId[" + planReportId + "] start run. TestPlanID:[" + testPlanReport.getTestPlanId() + "]. Execute api :" + JSONObject.toJSONString(executeApiCaseIdMap) + "; Execute scenario:" + JSONObject.toJSONString(executeScenarioCaseIdMap) + "; Execute performance:" + JSONObject.toJSONString(executePerformanceIdMap)); TestPlanReportExecuteCatch.updateApiTestPlanExecuteInfo(planReportId, executeApiCaseIdMap, executeScenarioCaseIdMap, executePerformanceIdMap); //执行接口案例任务 - this.executeApiTestCase(triggerMode, planReportId, new ArrayList<>(planApiCaseMap.keySet()), runModeConfig); - //执行场景执行任务 - this.executeScenarioCase(planReportId, testPlanID, projectID, runModeConfig, triggerMode, userId, planScenarioIdsMap); - return testPlanReport.getId(); - } + if(!planApiCaseMap.isEmpty()){ + this.executeApiTestCase(triggerMode, planReportId, new ArrayList<>(planApiCaseMap.keySet()), runModeConfig); + } - private void listenTaskExecuteStatus(String planReportId) { - executorService.submit(() -> { - try { - //10s 查询一次状态 - Thread.sleep(10000); - while (TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId) != null) { - testPlanReportService.countReport(planReportId); - Thread.sleep(10000); - } - } catch (InterruptedException e) { - TestPlanReportExecuteCatch.remove(planReportId); - e.printStackTrace(); - } - }); + //执行场景执行任务 + if(!planScenarioIdsMap.isEmpty()){ + this.executeScenarioCase(planReportId, testPlanReport.getTestPlanId(), projectID, runModeConfig, triggerMode, userId, planScenarioIdsMap); + } + testPlanReportService.startTestPlanExecuteListen(planReportId); } private void executeApiTestCase(String triggerMode, String planReportId, List planCaseIds, RunModeConfig runModeConfig) { - executorService.submit(() -> { - BatchRunDefinitionRequest request = new BatchRunDefinitionRequest(); - if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) { - request.setTriggerMode(ApiRunMode.JENKINS_API_PLAN.name()); - } else if (StringUtils.equals(triggerMode, ReportTriggerMode.MANUAL.name())) { - request.setTriggerMode(ApiRunMode.MANUAL_PLAN.name()); - } else { - request.setTriggerMode(ApiRunMode.SCHEDULE_API_PLAN.name()); - } - request.setPlanIds(planCaseIds); - request.setPlanReportId(planReportId); - request.setConfig(runModeConfig); - testPlanApiCaseService.run(request); - }); + BatchRunDefinitionRequest request = new BatchRunDefinitionRequest(); + if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) { + request.setTriggerMode(ApiRunMode.JENKINS_API_PLAN.name()); + } else if (StringUtils.equals(triggerMode, ReportTriggerMode.MANUAL.name())) { + request.setTriggerMode(ApiRunMode.MANUAL_PLAN.name()); + } else { + request.setTriggerMode(ApiRunMode.SCHEDULE_API_PLAN.name()); + } + + request.setPlanIds(planCaseIds); + request.setPlanReportId(planReportId); + request.setConfig(runModeConfig); + testPlanApiCaseService.run(request); } private void executeScenarioCase(String planReportId, String testPlanID, String projectID, RunModeConfig runModeConfig, String triggerMode, String userId, Map planScenarioIdMap) { - executorService.submit(() -> { - if (!planScenarioIdMap.isEmpty()) { - SchedulePlanScenarioExecuteRequest scenarioRequest = new SchedulePlanScenarioExecuteRequest(); - String senarionReportID = UUID.randomUUID().toString(); - scenarioRequest.setId(senarionReportID); - scenarioRequest.setReportId(senarionReportID); - scenarioRequest.setProjectId(projectID); - if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) { - scenarioRequest.setTriggerMode(ReportTriggerMode.API.name()); - scenarioRequest.setRunMode(ApiRunMode.JENKINS_SCENARIO_PLAN.name()); - } else if (StringUtils.equals(triggerMode, ReportTriggerMode.MANUAL.name())) { - scenarioRequest.setTriggerMode(ReportTriggerMode.MANUAL.name()); - scenarioRequest.setRunMode(ApiRunMode.JENKINS_SCENARIO_PLAN.name()); - } else { - scenarioRequest.setTriggerMode(ReportTriggerMode.SCHEDULE.name()); - scenarioRequest.setRunMode(ApiRunMode.SCHEDULE_SCENARIO_PLAN.name()); - } - scenarioRequest.setExecuteType(ExecuteType.Saved.name()); - Map> testPlanScenarioIdMap = new HashMap<>(); - testPlanScenarioIdMap.put(testPlanID, planScenarioIdMap); - scenarioRequest.setTestPlanScenarioIDMap(testPlanScenarioIdMap); - scenarioRequest.setReportUserID(userId); - scenarioRequest.setTestPlanID(testPlanID); - - scenarioRequest.setTestPlanReportId(planReportId); - - scenarioRequest.setConfig(runModeConfig); - this.scenarioRunModeConfig(scenarioRequest); + if (!planScenarioIdMap.isEmpty()) { + SchedulePlanScenarioExecuteRequest scenarioRequest = new SchedulePlanScenarioExecuteRequest(); + String senarionReportID = UUID.randomUUID().toString(); + scenarioRequest.setId(senarionReportID); + scenarioRequest.setReportId(senarionReportID); + scenarioRequest.setProjectId(projectID); + if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) { + scenarioRequest.setTriggerMode(ReportTriggerMode.API.name()); + scenarioRequest.setRunMode(ApiRunMode.JENKINS_SCENARIO_PLAN.name()); + } else if (StringUtils.equals(triggerMode, ReportTriggerMode.MANUAL.name())) { + scenarioRequest.setTriggerMode(ReportTriggerMode.MANUAL.name()); + scenarioRequest.setRunMode(ApiRunMode.JENKINS_SCENARIO_PLAN.name()); + } else { + scenarioRequest.setTriggerMode(ReportTriggerMode.SCHEDULE.name()); + scenarioRequest.setRunMode(ApiRunMode.SCHEDULE_SCENARIO_PLAN.name()); } - }); + scenarioRequest.setExecuteType(ExecuteType.Saved.name()); + Map> testPlanScenarioIdMap = new HashMap<>(); + testPlanScenarioIdMap.put(testPlanID, planScenarioIdMap); + scenarioRequest.setTestPlanScenarioIDMap(testPlanScenarioIdMap); + scenarioRequest.setReportUserID(userId); + scenarioRequest.setTestPlanID(testPlanID); + + scenarioRequest.setTestPlanReportId(planReportId); + + scenarioRequest.setConfig(runModeConfig); + this.scenarioRunModeConfig(scenarioRequest); + } } public String getLogDetails(String id) { @@ -1798,7 +1802,6 @@ public class TestPlanService { config = JSONObject.parseObject(reportConfig); } TestPlanSimpleReportDTO report = getReport(planId); - buildFunctionalReport(report, config, planId); buildApiReport(report, config, executeInfo, isFinish); buildLoadReport(report, config, executeInfo, planId, false); return report; diff --git a/backend/src/main/resources/application.properties b/backend/src/main/resources/application.properties index e41cece5fc..2a6486561a 100644 --- a/backend/src/main/resources/application.properties +++ b/backend/src/main/resources/application.properties @@ -12,7 +12,7 @@ server.ssl.key-alias=localhost # Hikari spring.datasource.type=com.zaxxer.hikari.HikariDataSource -spring.datasource.hikari.maximum-pool-size=100 +spring.datasource.hikari.maximum-pool-size=200 spring.datasource.hikari.auto-commit=true spring.datasource.hikari.idle-timeout=10000 spring.datasource.hikari.pool-name=DatebookHikariCP @@ -23,7 +23,7 @@ spring.datasource.hikari.connection-test-query=SELECT 1 spring.datasource.quartz.url=${spring.datasource.url} spring.datasource.quartz.username=${spring.datasource.username} spring.datasource.quartz.password=${spring.datasource.password} -spring.datasource.quartz.hikari.maximum-pool-size=50 +spring.datasource.quartz.hikari.maximum-pool-size=200 spring.datasource.quartz.hikari.auto-commit=true spring.datasource.quartz.hikari.idle-timeout=10000 spring.datasource.quartz.hikari.pool-name=DatebookHikariCP @@ -94,6 +94,8 @@ quartz.enabled=true quartz.scheduler-name=msServerJob quartz.thread-count=60 quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock=true +#schedule +testplan.thread-count=40 # file upload spring.servlet.multipart.max-file-size=500MB spring.servlet.multipart.max-request-size=500MB @@ -104,3 +106,4 @@ management.endpoints.web.exposure.include=* +