fix(测试计划): 优化测试计划执行

优化测试计划执行，更改配置信息

backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java

@@ -9,6 +9,7 @@ import io.metersphere.base.mapper.ApiScenarioReportMapper;
 import io.metersphere.commons.constants.TestPlanApiExecuteStatus;
 import io.metersphere.commons.constants.TestPlanResourceType;
 import io.metersphere.commons.utils.CommonBeanFactory;
+import io.metersphere.track.dto.TestPlanReportExecuteCheckResultDTO;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.commons.collections.CollectionUtils;
@@ -84,7 +85,8 @@ public class TestPlanExecuteInfo {
         }
     }
 
-    public synchronized int countUnFinishedNum() {
+    public synchronized TestPlanReportExecuteCheckResultDTO countUnFinishedNum() {
+        TestPlanReportExecuteCheckResultDTO executeCheck = new TestPlanReportExecuteCheckResultDTO();
         int unFinishedCount = 0;
 
         this.isApiCaseAllExecuted = true;
@@ -118,8 +120,21 @@ public class TestPlanExecuteInfo {
         if (lastUnFinishedNumCount != unFinishedCount) {
             lastUnFinishedNumCount = unFinishedCount;
             lastFinishedNumCountTime = System.currentTimeMillis();
+            executeCheck.setFinishedCaseChanged(true);
+        }else if(unFinishedCount == 0){
+            executeCheck.setFinishedCaseChanged(true);
+        }else {
+            executeCheck.setFinishedCaseChanged(false);
         }
-        return unFinishedCount;
+        executeCheck.setTimeOut(false);
+        if (unFinishedCount > 0) {
+            //20分钟没有案例执行结果更新，则定位超时
+            long nowTime = System.currentTimeMillis();
+            if (nowTime - lastFinishedNumCountTime > 1200000) {
+                executeCheck.setTimeOut(true);
+            }
+        }
+        return executeCheck;
     }
 
     public Map<String, Map<String, String>> getExecutedResult() {

backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java

@@ -1,14 +1,16 @@
 package io.metersphere.api.cache;
 
-import java.util.HashMap;
+import org.apache.commons.lang3.StringUtils;
-import java.util.Map;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * @author song.tianyang
  * @Date 2021/8/20 3:29 下午
  */
 public class TestPlanReportExecuteCatch {
-    private static Map<String, TestPlanExecuteInfo> testPlanReportMap = new HashMap<>();
+    private static Map<String, TestPlanExecuteInfo> testPlanReportMap = new ConcurrentHashMap<>();
 
     private TestPlanReportExecuteCatch() {
     }
@@ -44,8 +46,12 @@ public class TestPlanReportExecuteCatch {
     }
 
     public synchronized static boolean containsReport(String reportId){
+        if(StringUtils.isEmpty(reportId)){
+            return false;
+        }else {
             return testPlanReportMap != null && testPlanReportMap.containsKey(reportId);
         }
+    }
 
     public synchronized static void updateApiTestPlanExecuteInfo(String reportId,
                                                                  Map<String, String> apiCaseExecInfo, Map<String, String> apiScenarioCaseExecInfo, Map<String, String> loadCaseExecInfo) {
@@ -92,4 +98,12 @@ public class TestPlanReportExecuteCatch {
             testPlanReportMap.get(planReportId).finishAllTask();
         }
     }
+
+    public static Set<String> getAllReportId(){
+        if (testPlanReportMap != null) {
+            return testPlanReportMap.keySet();
+        }else {
+            return new HashSet<>();
+        }
+    }
 }

backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java

@@ -82,7 +82,7 @@ public class ApiJmeterFileService {
             }
             hashTree = apiAutomationService.generateHashTree(item, reportId, planEnvMap);
         }
-        return zipFilesToByteArray(remoteTestId, hashTree);
+        return zipFilesToByteArray(reportId, hashTree);
     }
 
     public byte[] downloadJmx(String runMode, String testId, String reportId, String testPlanScenarioId) {

backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java

@@ -3,6 +3,7 @@ package io.metersphere.job.sechedule;
 import io.metersphere.commons.constants.ReportTriggerMode;
 import io.metersphere.commons.constants.ScheduleGroup;
 import io.metersphere.commons.utils.CommonBeanFactory;
+import io.metersphere.commons.utils.LogUtil;
 import io.metersphere.track.service.TestPlanService;
 import org.quartz.*;
 
@@ -53,13 +54,14 @@ public class TestPlanTestJob extends MsScheduleJob {
         String runResourceId = this.resourceId;
         String runProjectId = this.projectID;
         String runUserId = this.userId;
+        LogUtil.info("Start test_plan_scehdule. test_plan_id:" + runProjectId);
         Thread thread = new Thread(new Runnable() {
             @Override
             public void run() {
                 testPlanService.run(runResourceId, runProjectId, runUserId, ReportTriggerMode.SCHEDULE.name(),config);
             }
         });
-        thread.run();
+        thread.start();
     }
 
     public static JobKey getJobKey(String testId) {

backend/src/main/java/io/metersphere/listener/AppStartListener.java

@@ -66,6 +66,10 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
     private TestReviewTestCaseService testReviewTestCaseService;
     @Resource
     private MockConfigService mockConfigService;
+    @Resource
+    private TestPlanService testPlanService;
+    @Resource
+    private TestPlanReportService testPlanReportService;
 
     @Value("${jmeter.home}")
     private String jmeterHome;
@@ -77,6 +81,8 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
     private String quartzScheduleName;
     @Value("${quartz.thread-count}")
     private int quartzThreadCount;
+    @Value("${testplan.thread-count}")
+    private int testplanExecuteThreadPool;
 
     @Override
     public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
@@ -85,6 +91,10 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
 
         System.setProperty("jmeter.home", jmeterHome);
 
+        //修改需要自定义的线程池
+        testPlanService.resetThreadPool(testplanExecuteThreadPool);
+        testPlanReportService.resetThreadPool(testplanExecuteThreadPool);
+
         loadJars();
 
         initPythonEnv();
@@ -105,7 +115,10 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
                 "quartz.acquireTriggersWithinLock :" + acquireTriggersWithinLock + "\r\n" +
                 "quartz.enabled :" + quartzEnable + "\r\n" +
                 "quartz.scheduler-name :" + quartzScheduleName + "\r\n" +
-                "quartz.thread-count :" + quartzThreadCount + "\r\n");
+                "quartz.thread-count :" + quartzThreadCount + "\r\n" +
+                "testplan.execute.thread.pool :" + testplanExecuteThreadPool + "\r\n"
+        );
+
         scheduleService.startEnableSchedules();
 
     }

backend/src/main/java/io/metersphere/track/dto/TestPlanReportExecuteCheckResultDTO.java

@@ -0,0 +1,11 @@
+package io.metersphere.track.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class TestPlanReportExecuteCheckResultDTO {
+    private boolean isTimeOut;
+    private boolean isFinishedCaseChanged;
+}

backend/src/main/java/io/metersphere/track/service/TestPlanReportService.java

@@ -47,6 +47,7 @@ import javax.annotation.Resource;
 import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.stream.Collectors;
 
 /**
@@ -104,9 +105,16 @@ public class TestPlanReportService {
     private UserService userService;
     @Resource
     private ProjectService projectService;
-
     private final ExecutorService executorService = Executors.newFixedThreadPool(20, new NamedThreadFactory("TestPlanReportService"));
 
+    private final ThreadPoolExecutor planListenerExecutorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(20, new NamedThreadFactory("TestPlanExecuteListen"));
+
+    public void resetThreadPool(int threadCount){
+        if(threadCount > 0){
+            planListenerExecutorService.setMaximumPoolSize(threadCount);
+        }
+    }
+
     public List<TestPlanReportDTO> list(QueryTestPlanReportRequest request) {
         List<TestPlanReportDTO> list = new ArrayList<>();
         request.setOrders(ServiceUtils.getDefaultOrder(request.getOrders()));
@@ -584,8 +592,6 @@ public class TestPlanReportService {
         boolean scenarioIsOk = executeInfo.isScenarioAllExecuted();
         boolean performanceIsOk = executeInfo.isLoadCaseAllExecuted();
 
-        testPlanLog.info("ReportId[" + testPlanReport.getId() + "] count over. Testplan Execute Result:  Api is over ->" + apiCaseIsOk + "; scenario is over ->" + scenarioIsOk + "; performance is over ->" + performanceIsOk);
-
         if (apiCaseIsOk) {
             testPlanReport.setIsApiCaseExecuting(false);
         }
@@ -1113,13 +1119,17 @@ public class TestPlanReportService {
     }
 
     public void countReport(String planReportId) {
-        boolean isTimeOut = this.checkTestPlanReportIsTimeOut(planReportId);
+        TestPlanReportExecuteCheckResultDTO checkResult = this.checkTestPlanReportIsTimeOut(planReportId);
-        if (isTimeOut) {
+        testPlanLog.info("Check PlanReport:" + planReportId + "; result: "+ JSON.toJSONString(checkResult));
+        if (checkResult.isTimeOut()) {
             //判断是否超时。超时时强行停止任务
             TestPlanReportExecuteCatch.finishAllTask(planReportId);
+            checkResult.setFinishedCaseChanged(true);
         }
+        if(checkResult.isFinishedCaseChanged()){
             this.updateExecuteApis(planReportId);
         }
+    }
 
     public TestPlanSimpleReportDTO getReport(String reportId) {
         TestPlanReportContentExample example = new TestPlanReportContentExample();
@@ -1254,7 +1264,7 @@ public class TestPlanReportService {
         testPlanReportContentMapper.updateByExampleSelective(bloBs, example);
     }
 
-    private boolean checkTestPlanReportIsTimeOut(String planReportId) {
+    private TestPlanReportExecuteCheckResultDTO checkTestPlanReportIsTimeOut(String planReportId) {
         //同步数据库更新状态信息
         try {
             this.syncReportStatus(planReportId);
@@ -1264,16 +1274,9 @@ public class TestPlanReportService {
         }
 
         TestPlanExecuteInfo executeInfo = TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId);
-        int unFinishNum = executeInfo.countUnFinishedNum();
+        TestPlanReportExecuteCheckResultDTO checkResult = executeInfo.countUnFinishedNum();
-        if (unFinishNum > 0) {
+
-            //20分钟没有案例执行结果更新，则定位超时
+        return checkResult;
-            long lastCountTime = executeInfo.getLastFinishedNumCountTime();
-            long nowTime = System.currentTimeMillis();
-            if (nowTime - lastCountTime > 1200000) {
-                return true;
-            }
-        }
-        return false;
     }
 
     private void syncReportStatus(String planReportId) {
@@ -1331,4 +1334,26 @@ public class TestPlanReportService {
         TestPlanReportExecuteCatch.remove(planReportId);
     }
 
+    public void startTestPlanExecuteListen(String reportId){
+        planListenerExecutorService.submit(() -> {
+            while (TestPlanReportExecuteCatch.containsReport(reportId)){
+                //检查是否存在数据库里
+                TestPlanReportExample example = new TestPlanReportExample();
+                example.createCriteria().andIdEqualTo(reportId);
+                long count = testPlanReportMapper.countByExample(example);
+                if(count > 0){
+                    testPlanLog.info("Start check testPlanReport:" + reportId);
+                    countReport(reportId);
+                    try {
+                        Thread.sleep(10000);
+                    }catch (Exception e){
+                        LogUtil.info(e);
+                    }
+                }else {
+                    TestPlanReportExecuteCatch.remove(reportId);
+                }
+            }
+        });
+    }
+
 }

backend/src/main/java/io/metersphere/track/service/TestPlanService.java

@@ -70,7 +70,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.CollectionUtils;
 
@@ -84,8 +83,8 @@ import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.text.SimpleDateFormat;
 import java.util.*;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -190,7 +189,13 @@ public class TestPlanService {
     @Resource
     private TestPlanFollowMapper testPlanFollowMapper;
 
-    private final ExecutorService executorService = Executors.newFixedThreadPool(40, new NamedThreadFactory("TestPlanService"));
+    private final ThreadPoolExecutor executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(20, new NamedThreadFactory("TestPlanService"));
+
+    public void resetThreadPool(int threadCount){
+        if(threadCount > 0){
+            executorService.setMaximumPoolSize(threadCount);
+        }
+    }
 
     public synchronized TestPlan addTestPlan(AddTestPlanRequest testPlan) {
         if (getTestPlanByName(testPlan.getName()).size() > 0) {
@@ -1095,7 +1100,6 @@ public class TestPlanService {
         TestPlanReport testPlanReport = reportInfoDTO.getTestPlanReport();
         Map<String, String> planScenarioIdsMap = reportInfoDTO.getPlanScenarioIdMap();
         Map<String, String> planApiCaseMap = reportInfoDTO.getApiTestCaseDataMap();
-        Map<String, String> performanceIdMap = reportInfoDTO.getPerformanceIdMap();
 
         if (runModeConfig.getMode().equals(RunModeConstants.PARALLEL.toString())) {
             // 校验并发数量
@@ -1116,8 +1120,22 @@ public class TestPlanService {
         extTestPlanMapper.updateActualEndTimeIsNullById(testPlanID);
         String planReportId = testPlanReport.getId();
         testPlanLog.info("ReportId[" + planReportId + "] created. TestPlanID:[" + testPlanID + "]. " + "API Run Config:【" + apiRunConfig + "】");
-        //开启测试计划执行状态的监听
+
-        this.listenTaskExecuteStatus(planReportId);
+        RunModeConfig finalRunModeConfig = runModeConfig;
+        executorService.submit(() -> {
+            testPlanLog.info("ReportId[" + planReportId + "] start execute.");
+            this.executeTestPlan(reportInfoDTO, triggerMode, projectID, userId, finalRunModeConfig);
+            testPlanLog.info("ReportId[" + planReportId + "] is executing.");
+        });
+        return testPlanReport.getId();
+    }
+
+    private void executeTestPlan(TestPlanScheduleReportInfoDTO reportInfoDTO, String triggerMode, String projectID, String userId, RunModeConfig runModeConfig) {
+        TestPlanReport testPlanReport = reportInfoDTO.getTestPlanReport();
+        String planReportId = testPlanReport.getId();
+        Map<String, String> planScenarioIdsMap = reportInfoDTO.getPlanScenarioIdMap();
+        Map<String, String> planApiCaseMap = reportInfoDTO.getApiTestCaseDataMap();
+        Map<String, String> performanceIdMap = reportInfoDTO.getPerformanceIdMap();
 
         //不同任务的执行ID
         Map<String, String> executePerformanceIdMap = new HashMap<>();
@@ -1180,33 +1198,22 @@ public class TestPlanService {
         for (String id : planScenarioIdsMap.keySet()) {
             executeScenarioCaseIdMap.put(id, TestPlanApiExecuteStatus.RUNNING.name());
         }
-        testPlanLog.info("ReportId[" + planReportId + "] start run. TestPlanID:[" + testPlanID + "].  Execute api :" + JSONObject.toJSONString(executeApiCaseIdMap) + "; Execute scenario:" + JSONObject.toJSONString(executeScenarioCaseIdMap) + "; Execute performance:" + JSONObject.toJSONString(executePerformanceIdMap));
+        testPlanLog.info("ReportId[" + planReportId + "] start run. TestPlanID:[" + testPlanReport.getTestPlanId() + "].  Execute api :" + JSONObject.toJSONString(executeApiCaseIdMap) + "; Execute scenario:" + JSONObject.toJSONString(executeScenarioCaseIdMap) + "; Execute performance:" + JSONObject.toJSONString(executePerformanceIdMap));
         TestPlanReportExecuteCatch.updateApiTestPlanExecuteInfo(planReportId, executeApiCaseIdMap, executeScenarioCaseIdMap, executePerformanceIdMap);
         //执行接口案例任务
+        if(!planApiCaseMap.isEmpty()){
             this.executeApiTestCase(triggerMode, planReportId, new ArrayList<>(planApiCaseMap.keySet()), runModeConfig);
-        //执行场景执行任务
-        this.executeScenarioCase(planReportId, testPlanID, projectID, runModeConfig, triggerMode, userId, planScenarioIdsMap);
-        return testPlanReport.getId();
         }
 
-    private void listenTaskExecuteStatus(String planReportId) {
+        //执行场景执行任务
-        executorService.submit(() -> {
+        if(!planScenarioIdsMap.isEmpty()){
-            try {
+            this.executeScenarioCase(planReportId, testPlanReport.getTestPlanId(), projectID, runModeConfig, triggerMode, userId, planScenarioIdsMap);
-                //10s 查询一次状态
-                Thread.sleep(10000);
-                while (TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId) != null) {
-                    testPlanReportService.countReport(planReportId);
-                    Thread.sleep(10000);
         }
-            } catch (InterruptedException e) {
+        testPlanReportService.startTestPlanExecuteListen(planReportId);
-                TestPlanReportExecuteCatch.remove(planReportId);
-                e.printStackTrace();
-            }
-        });
     }
 
     private void executeApiTestCase(String triggerMode, String planReportId, List<String> planCaseIds, RunModeConfig runModeConfig) {
-        executorService.submit(() -> {
+
         BatchRunDefinitionRequest request = new BatchRunDefinitionRequest();
         if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) {
             request.setTriggerMode(ApiRunMode.JENKINS_API_PLAN.name());
@@ -1220,11 +1227,9 @@ public class TestPlanService {
         request.setPlanReportId(planReportId);
         request.setConfig(runModeConfig);
         testPlanApiCaseService.run(request);
-        });
     }
 
     private void executeScenarioCase(String planReportId, String testPlanID, String projectID, RunModeConfig runModeConfig, String triggerMode, String userId, Map<String, String> planScenarioIdMap) {
-        executorService.submit(() -> {
         if (!planScenarioIdMap.isEmpty()) {
             SchedulePlanScenarioExecuteRequest scenarioRequest = new SchedulePlanScenarioExecuteRequest();
             String senarionReportID = UUID.randomUUID().toString();
@@ -1253,7 +1258,6 @@ public class TestPlanService {
             scenarioRequest.setConfig(runModeConfig);
             this.scenarioRunModeConfig(scenarioRequest);
         }
-        });
     }
 
     public String getLogDetails(String id) {
@@ -1798,7 +1802,6 @@ public class TestPlanService {
                 config = JSONObject.parseObject(reportConfig);
             }
             TestPlanSimpleReportDTO report = getReport(planId);
-            buildFunctionalReport(report, config, planId);
             buildApiReport(report, config, executeInfo, isFinish);
             buildLoadReport(report, config, executeInfo, planId, false);
             return report;

backend/src/main/resources/application.properties

@@ -12,7 +12,7 @@ server.ssl.key-alias=localhost
 
 # Hikari
 spring.datasource.type=com.zaxxer.hikari.HikariDataSource
-spring.datasource.hikari.maximum-pool-size=100
+spring.datasource.hikari.maximum-pool-size=200
 spring.datasource.hikari.auto-commit=true
 spring.datasource.hikari.idle-timeout=10000
 spring.datasource.hikari.pool-name=DatebookHikariCP
@@ -23,7 +23,7 @@ spring.datasource.hikari.connection-test-query=SELECT 1
 spring.datasource.quartz.url=${spring.datasource.url}
 spring.datasource.quartz.username=${spring.datasource.username}
 spring.datasource.quartz.password=${spring.datasource.password}
-spring.datasource.quartz.hikari.maximum-pool-size=50
+spring.datasource.quartz.hikari.maximum-pool-size=200
 spring.datasource.quartz.hikari.auto-commit=true
 spring.datasource.quartz.hikari.idle-timeout=10000
 spring.datasource.quartz.hikari.pool-name=DatebookHikariCP
@@ -94,6 +94,8 @@ quartz.enabled=true
 quartz.scheduler-name=msServerJob
 quartz.thread-count=60
 quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock=true
+#schedule
+testplan.thread-count=40
 # file upload
 spring.servlet.multipart.max-file-size=500MB
 spring.servlet.multipart.max-request-size=500MB
@@ -104,3 +106,4 @@ management.endpoints.web.exposure.include=*
 
 
 
+