test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix(测试计划): 优化测试计划执行 

优化测试计划执行，更改配置信息
This commit is contained in:
song-tianyang 2021-12-17 16:06:58 +08:00 committed by song-tianyang
parent 5b4e8a836b
commit b7cd21bc4a
9 changed files with 184 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
backend/src/main/java/io/metersphere/api/cache/TestPlanExecuteInfo.java vendored
View File

@ -9,6 +9,7 @@ import io.metersphere.base.mapper.ApiScenarioReportMapper;
import io.metersphere.commons.constants.TestPlanApiExecuteStatus;
import io.metersphere.commons.constants.TestPlanResourceType;
import io.metersphere.commons.utils.CommonBeanFactory;
import io.metersphere.track.dto.TestPlanReportExecuteCheckResultDTO;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.collections.CollectionUtils;
@ -84,7 +85,8 @@ public class TestPlanExecuteInfo {
}
}
public synchronized int countUnFinishedNum() {
public synchronized TestPlanReportExecuteCheckResultDTO countUnFinishedNum() {
TestPlanReportExecuteCheckResultDTO executeCheck = new TestPlanReportExecuteCheckResultDTO();
int unFinishedCount = 0;
this.isApiCaseAllExecuted = true;
@ -118,8 +120,21 @@ public class TestPlanExecuteInfo {
if (lastUnFinishedNumCount != unFinishedCount) {
lastUnFinishedNumCount = unFinishedCount;
lastFinishedNumCountTime = System.currentTimeMillis();
executeCheck.setFinishedCaseChanged(true);
}else if(unFinishedCount == 0){
executeCheck.setFinishedCaseChanged(true);
}else {
executeCheck.setFinishedCaseChanged(false);
}
return unFinishedCount;
executeCheck.setTimeOut(false);
if (unFinishedCount > 0) {
//20分钟没有案例执行结果更新，则定位超时
long nowTime = System.currentTimeMillis();
if (nowTime - lastFinishedNumCountTime > 1200000) {
executeCheck.setTimeOut(true);
}
}
return executeCheck;
}
public Map<String, Map<String, String>> getExecutedResult() {

20
backend/src/main/java/io/metersphere/api/cache/TestPlanReportExecuteCatch.java vendored
View File

@ -1,14 +1,16 @@
package io.metersphere.api.cache;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author song.tianyang
* @Date 2021/8/20 3:29 下午
*/
public class TestPlanReportExecuteCatch {
private static Map<String, TestPlanExecuteInfo> testPlanReportMap = new HashMap<>();
private static Map<String, TestPlanExecuteInfo> testPlanReportMap = new ConcurrentHashMap<>();
private TestPlanReportExecuteCatch() {
}
@ -44,8 +46,12 @@ public class TestPlanReportExecuteCatch {
}
public synchronized static boolean containsReport(String reportId){
if(StringUtils.isEmpty(reportId)){
return false;
}else {
return testPlanReportMap != null && testPlanReportMap.containsKey(reportId);
}
}
public synchronized static void updateApiTestPlanExecuteInfo(String reportId,
Map<String, String> apiCaseExecInfo, Map<String, String> apiScenarioCaseExecInfo, Map<String, String> loadCaseExecInfo) {
@ -92,4 +98,12 @@ public class TestPlanReportExecuteCatch {
testPlanReportMap.get(planReportId).finishAllTask();
}
}
public static Set<String> getAllReportId(){
if (testPlanReportMap != null) {
return testPlanReportMap.keySet();
}else {
return new HashSet<>();
}
}
}

2
backend/src/main/java/io/metersphere/api/service/ApiJmeterFileService.java
View File

@ -82,7 +82,7 @@ public class ApiJmeterFileService {
}
hashTree = apiAutomationService.generateHashTree(item, reportId, planEnvMap);
}
return zipFilesToByteArray(remoteTestId, hashTree);
return zipFilesToByteArray(reportId, hashTree);
}
public byte[] downloadJmx(String runMode, String testId, String reportId, String testPlanScenarioId) {

4
backend/src/main/java/io/metersphere/job/sechedule/TestPlanTestJob.java
View File

@ -3,6 +3,7 @@ package io.metersphere.job.sechedule;
import io.metersphere.commons.constants.ReportTriggerMode;
import io.metersphere.commons.constants.ScheduleGroup;
import io.metersphere.commons.utils.CommonBeanFactory;
import io.metersphere.commons.utils.LogUtil;
import io.metersphere.track.service.TestPlanService;
import org.quartz.*;
@ -53,13 +54,14 @@ public class TestPlanTestJob extends MsScheduleJob {
String runResourceId = this.resourceId;
String runProjectId = this.projectID;
String runUserId = this.userId;
LogUtil.info("Start test_plan_scehdule. test_plan_id:" + runProjectId);
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
testPlanService.run(runResourceId, runProjectId, runUserId, ReportTriggerMode.SCHEDULE.name(),config);
}
});
thread.run();
thread.start();
}
public static JobKey getJobKey(String testId) {

15
backend/src/main/java/io/metersphere/listener/AppStartListener.java
View File

@ -66,6 +66,10 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
private TestReviewTestCaseService testReviewTestCaseService;
@Resource
private MockConfigService mockConfigService;
@Resource
private TestPlanService testPlanService;
@Resource
private TestPlanReportService testPlanReportService;
@Value("${jmeter.home}")
private String jmeterHome;
@ -77,6 +81,8 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
private String quartzScheduleName;
@Value("${quartz.thread-count}")
private int quartzThreadCount;
@Value("${testplan.thread-count}")
private int testplanExecuteThreadPool;
@Override
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
@ -85,6 +91,10 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
System.setProperty("jmeter.home", jmeterHome);
//修改需要自定义的线程池
testPlanService.resetThreadPool(testplanExecuteThreadPool);
testPlanReportService.resetThreadPool(testplanExecuteThreadPool);
loadJars();
initPythonEnv();
@ -105,7 +115,10 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
"quartz.acquireTriggersWithinLock :" + acquireTriggersWithinLock + "\r\n" +
"quartz.enabled :" + quartzEnable + "\r\n" +
"quartz.scheduler-name :" + quartzScheduleName + "\r\n" +
"quartz.thread-count :" + quartzThreadCount + "\r\n");
"quartz.thread-count :" + quartzThreadCount + "\r\n" +
"testplan.execute.thread.pool :" + testplanExecuteThreadPool + "\r\n"
);
scheduleService.startEnableSchedules();
}

11
backend/src/main/java/io/metersphere/track/dto/TestPlanReportExecuteCheckResultDTO.java Normal file
View File

@ -0,0 +1,11 @@
package io.metersphere.track.dto;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class TestPlanReportExecuteCheckResultDTO {
private boolean isTimeOut;
private boolean isFinishedCaseChanged;
}

57
backend/src/main/java/io/metersphere/track/service/TestPlanReportService.java
View File

@ -47,6 +47,7 @@ import javax.annotation.Resource;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.stream.Collectors;
/**
@ -104,9 +105,16 @@ public class TestPlanReportService {
private UserService userService;
@Resource
private ProjectService projectService;
private final ExecutorService executorService = Executors.newFixedThreadPool(20, new NamedThreadFactory("TestPlanReportService"));
private final ThreadPoolExecutor planListenerExecutorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(20, new NamedThreadFactory("TestPlanExecuteListen"));
public void resetThreadPool(int threadCount){
if(threadCount > 0){
planListenerExecutorService.setMaximumPoolSize(threadCount);
}
}
public List<TestPlanReportDTO> list(QueryTestPlanReportRequest request) {
List<TestPlanReportDTO> list = new ArrayList<>();
request.setOrders(ServiceUtils.getDefaultOrder(request.getOrders()));
@ -584,8 +592,6 @@ public class TestPlanReportService {
boolean scenarioIsOk = executeInfo.isScenarioAllExecuted();
boolean performanceIsOk = executeInfo.isLoadCaseAllExecuted();
testPlanLog.info("ReportId[" + testPlanReport.getId() + "] count over. Testplan Execute Result: Api is over ->" + apiCaseIsOk + "; scenario is over ->" + scenarioIsOk + "; performance is over ->" + performanceIsOk);
if (apiCaseIsOk) {
testPlanReport.setIsApiCaseExecuting(false);
}
@ -1113,13 +1119,17 @@ public class TestPlanReportService {
}
public void countReport(String planReportId) {
boolean isTimeOut = this.checkTestPlanReportIsTimeOut(planReportId);
if (isTimeOut) {
TestPlanReportExecuteCheckResultDTO checkResult = this.checkTestPlanReportIsTimeOut(planReportId);
testPlanLog.info("Check PlanReport:" + planReportId + "; result: "+ JSON.toJSONString(checkResult));
if (checkResult.isTimeOut()) {
//判断是否超时。超时时强行停止任务
TestPlanReportExecuteCatch.finishAllTask(planReportId);
checkResult.setFinishedCaseChanged(true);
}
if(checkResult.isFinishedCaseChanged()){
this.updateExecuteApis(planReportId);
}
}
public TestPlanSimpleReportDTO getReport(String reportId) {
TestPlanReportContentExample example = new TestPlanReportContentExample();
@ -1254,7 +1264,7 @@ public class TestPlanReportService {
testPlanReportContentMapper.updateByExampleSelective(bloBs, example);
}
private boolean checkTestPlanReportIsTimeOut(String planReportId) {
private TestPlanReportExecuteCheckResultDTO checkTestPlanReportIsTimeOut(String planReportId) {
//同步数据库更新状态信息
try {
this.syncReportStatus(planReportId);
@ -1264,16 +1274,9 @@ public class TestPlanReportService {
}
TestPlanExecuteInfo executeInfo = TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId);
int unFinishNum = executeInfo.countUnFinishedNum();
if (unFinishNum > 0) {
//20分钟没有案例执行结果更新，则定位超时
long lastCountTime = executeInfo.getLastFinishedNumCountTime();
long nowTime = System.currentTimeMillis();
if (nowTime - lastCountTime > 1200000) {
return true;
}
}
return false;
TestPlanReportExecuteCheckResultDTO checkResult = executeInfo.countUnFinishedNum();
return checkResult;
}
private void syncReportStatus(String planReportId) {
@ -1331,4 +1334,26 @@ public class TestPlanReportService {
TestPlanReportExecuteCatch.remove(planReportId);
}
public void startTestPlanExecuteListen(String reportId){
planListenerExecutorService.submit(() -> {
while (TestPlanReportExecuteCatch.containsReport(reportId)){
//检查是否存在数据库里
TestPlanReportExample example = new TestPlanReportExample();
example.createCriteria().andIdEqualTo(reportId);
long count = testPlanReportMapper.countByExample(example);
if(count > 0){
testPlanLog.info("Start check testPlanReport:" + reportId);
countReport(reportId);
try {
Thread.sleep(10000);
}catch (Exception e){
LogUtil.info(e);
}
}else {
TestPlanReportExecuteCatch.remove(reportId);
}
}
});
}
}

59
backend/src/main/java/io/metersphere/track/service/TestPlanService.java
View File

@ -70,7 +70,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
@ -84,8 +83,8 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -190,7 +189,13 @@ public class TestPlanService {
@Resource
private TestPlanFollowMapper testPlanFollowMapper;
private final ExecutorService executorService = Executors.newFixedThreadPool(40, new NamedThreadFactory("TestPlanService"));
private final ThreadPoolExecutor executorService = (ThreadPoolExecutor) Executors.newFixedThreadPool(20, new NamedThreadFactory("TestPlanService"));
public void resetThreadPool(int threadCount){
if(threadCount > 0){
executorService.setMaximumPoolSize(threadCount);
}
}
public synchronized TestPlan addTestPlan(AddTestPlanRequest testPlan) {
if (getTestPlanByName(testPlan.getName()).size() > 0) {
@ -1095,7 +1100,6 @@ public class TestPlanService {
TestPlanReport testPlanReport = reportInfoDTO.getTestPlanReport();
Map<String, String> planScenarioIdsMap = reportInfoDTO.getPlanScenarioIdMap();
Map<String, String> planApiCaseMap = reportInfoDTO.getApiTestCaseDataMap();
Map<String, String> performanceIdMap = reportInfoDTO.getPerformanceIdMap();
if (runModeConfig.getMode().equals(RunModeConstants.PARALLEL.toString())) {
// 校验并发数量
@ -1116,8 +1120,22 @@ public class TestPlanService {
extTestPlanMapper.updateActualEndTimeIsNullById(testPlanID);
String planReportId = testPlanReport.getId();
testPlanLog.info("ReportId[" + planReportId + "] created. TestPlanID:[" + testPlanID + "]. " + "API Run Config:【" + apiRunConfig + "】");
//开启测试计划执行状态的监听
this.listenTaskExecuteStatus(planReportId);
RunModeConfig finalRunModeConfig = runModeConfig;
executorService.submit(() -> {
testPlanLog.info("ReportId[" + planReportId + "] start execute.");
this.executeTestPlan(reportInfoDTO, triggerMode, projectID, userId, finalRunModeConfig);
testPlanLog.info("ReportId[" + planReportId + "] is executing.");
});
return testPlanReport.getId();
}
private void executeTestPlan(TestPlanScheduleReportInfoDTO reportInfoDTO, String triggerMode, String projectID, String userId, RunModeConfig runModeConfig) {
TestPlanReport testPlanReport = reportInfoDTO.getTestPlanReport();
String planReportId = testPlanReport.getId();
Map<String, String> planScenarioIdsMap = reportInfoDTO.getPlanScenarioIdMap();
Map<String, String> planApiCaseMap = reportInfoDTO.getApiTestCaseDataMap();
Map<String, String> performanceIdMap = reportInfoDTO.getPerformanceIdMap();
//不同任务的执行ID
Map<String, String> executePerformanceIdMap = new HashMap<>();
@ -1180,33 +1198,22 @@ public class TestPlanService {
for (String id : planScenarioIdsMap.keySet()) {
executeScenarioCaseIdMap.put(id, TestPlanApiExecuteStatus.RUNNING.name());
}
testPlanLog.info("ReportId[" + planReportId + "] start run. TestPlanID:[" + testPlanID + "]. Execute api :" + JSONObject.toJSONString(executeApiCaseIdMap) + "; Execute scenario:" + JSONObject.toJSONString(executeScenarioCaseIdMap) + "; Execute performance:" + JSONObject.toJSONString(executePerformanceIdMap));
testPlanLog.info("ReportId[" + planReportId + "] start run. TestPlanID:[" + testPlanReport.getTestPlanId() + "]. Execute api :" + JSONObject.toJSONString(executeApiCaseIdMap) + "; Execute scenario:" + JSONObject.toJSONString(executeScenarioCaseIdMap) + "; Execute performance:" + JSONObject.toJSONString(executePerformanceIdMap));
TestPlanReportExecuteCatch.updateApiTestPlanExecuteInfo(planReportId, executeApiCaseIdMap, executeScenarioCaseIdMap, executePerformanceIdMap);
//执行接口案例任务
if(!planApiCaseMap.isEmpty()){
this.executeApiTestCase(triggerMode, planReportId, new ArrayList<>(planApiCaseMap.keySet()), runModeConfig);
//执行场景执行任务
this.executeScenarioCase(planReportId, testPlanID, projectID, runModeConfig, triggerMode, userId, planScenarioIdsMap);
return testPlanReport.getId();
}
private void listenTaskExecuteStatus(String planReportId) {
executorService.submit(() -> {
try {
//10s 查询一次状态
Thread.sleep(10000);
while (TestPlanReportExecuteCatch.getTestPlanExecuteInfo(planReportId) != null) {
testPlanReportService.countReport(planReportId);
Thread.sleep(10000);
//执行场景执行任务
if(!planScenarioIdsMap.isEmpty()){
this.executeScenarioCase(planReportId, testPlanReport.getTestPlanId(), projectID, runModeConfig, triggerMode, userId, planScenarioIdsMap);
}
} catch (InterruptedException e) {
TestPlanReportExecuteCatch.remove(planReportId);
e.printStackTrace();
}
});
testPlanReportService.startTestPlanExecuteListen(planReportId);
}
private void executeApiTestCase(String triggerMode, String planReportId, List<String> planCaseIds, RunModeConfig runModeConfig) {
executorService.submit(() -> {
BatchRunDefinitionRequest request = new BatchRunDefinitionRequest();
if (StringUtils.equals(triggerMode, ReportTriggerMode.API.name())) {
request.setTriggerMode(ApiRunMode.JENKINS_API_PLAN.name());
@ -1220,11 +1227,9 @@ public class TestPlanService {
request.setPlanReportId(planReportId);
request.setConfig(runModeConfig);
testPlanApiCaseService.run(request);
});
}
private void executeScenarioCase(String planReportId, String testPlanID, String projectID, RunModeConfig runModeConfig, String triggerMode, String userId, Map<String, String> planScenarioIdMap) {
executorService.submit(() -> {
if (!planScenarioIdMap.isEmpty()) {
SchedulePlanScenarioExecuteRequest scenarioRequest = new SchedulePlanScenarioExecuteRequest();
String senarionReportID = UUID.randomUUID().toString();
@ -1253,7 +1258,6 @@ public class TestPlanService {
scenarioRequest.setConfig(runModeConfig);
this.scenarioRunModeConfig(scenarioRequest);
}
});
}
public String getLogDetails(String id) {
@ -1798,7 +1802,6 @@ public class TestPlanService {
config = JSONObject.parseObject(reportConfig);
}
TestPlanSimpleReportDTO report = getReport(planId);
buildFunctionalReport(report, config, planId);
buildApiReport(report, config, executeInfo, isFinish);
buildLoadReport(report, config, executeInfo, planId, false);
return report;

7
backend/src/main/resources/application.properties
View File

@ -12,7 +12,7 @@ server.ssl.key-alias=localhost
# Hikari
spring.datasource.type=com.zaxxer.hikari.HikariDataSource
spring.datasource.hikari.maximum-pool-size=100
spring.datasource.hikari.maximum-pool-size=200
spring.datasource.hikari.auto-commit=true
spring.datasource.hikari.idle-timeout=10000
spring.datasource.hikari.pool-name=DatebookHikariCP
@ -23,7 +23,7 @@ spring.datasource.hikari.connection-test-query=SELECT 1
spring.datasource.quartz.url=${spring.datasource.url}
spring.datasource.quartz.username=${spring.datasource.username}
spring.datasource.quartz.password=${spring.datasource.password}
spring.datasource.quartz.hikari.maximum-pool-size=50
spring.datasource.quartz.hikari.maximum-pool-size=200
spring.datasource.quartz.hikari.auto-commit=true
spring.datasource.quartz.hikari.idle-timeout=10000
spring.datasource.quartz.hikari.pool-name=DatebookHikariCP
@ -94,6 +94,8 @@ quartz.enabled=true
quartz.scheduler-name=msServerJob
quartz.thread-count=60
quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock=true
#schedule
testplan.thread-count=40
# file upload
spring.servlet.multipart.max-file-size=500MB
spring.servlet.multipart.max-request-size=500MB
@ -104,3 +106,4 @@ management.endpoints.web.exposure.include=*