test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix (接口自动化): 并发执行优化

This commit is contained in:
fit2-zhao 2021-08-30 17:28:28 +08:00 committed by fit2-zhao
parent 59b1faec6e
commit 4443a88dca
14 changed files with 266 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
backend/src/main/java/io/metersphere/api/dto/RunModeDataDTO.java
View File

@ -1,10 +1,13 @@
package io.metersphere.api.dto; package io.metersphere.api.dto;
import io.metersphere.api.dto.automation.APIScenarioReportResult; import io.metersphere.api.dto.automation.APIScenarioReportResult;
import io.metersphere.base.domain.ApiScenarioWithBLOBs;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.apache.jorphan.collections.HashTree; import org.apache.jorphan.collections.HashTree;
import java.util.Map;
@Getter @Getter
@Setter @Setter
public class RunModeDataDTO { public class RunModeDataDTO {
@ -17,6 +20,11 @@ public class RunModeDataDTO {
// //
private String apiCaseId; private String apiCaseId;
private ApiScenarioWithBLOBs scenario;
private Map<String, String> planEnvMap;
public RunModeDataDTO(){
}
public RunModeDataDTO(String testId, String apiCaseId) { public RunModeDataDTO(String testId, String apiCaseId) {
this.testId = testId; this.testId = testId;
this.apiCaseId = apiCaseId; this.apiCaseId = apiCaseId;
@ -36,4 +44,10 @@ public class RunModeDataDTO {
this.hashTree = hashTree; this.hashTree = hashTree;
this.report = report; this.report = report;
} }
public RunModeDataDTO(HashTree hashTree, APIScenarioReportResult report, String testId) {
this.hashTree = hashTree;
this.report = report;
this.testId = testId;
}
} }

4
backend/src/main/java/io/metersphere/api/jmeter/APIBackendListenerHandler.java
View File

@ -7,6 +7,8 @@ import io.metersphere.api.service.MsResultService;
import io.metersphere.api.service.TestResultService; import io.metersphere.api.service.TestResultService;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.jmeter.samplers.SampleResult; import org.apache.jmeter.samplers.SampleResult;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.Comparator; import java.util.Comparator;
@ -18,6 +20,8 @@ import java.util.Map;
* 获取结果和数据库操作分离 * 获取结果和数据库操作分离
* 减少占用的数据库连接 * 减少占用的数据库连接
*/ */
@Service
@Transactional(rollbackFor = Exception.class)
public class APIBackendListenerHandler { public class APIBackendListenerHandler {
@Resource @Resource

53
backend/src/main/java/io/metersphere/api/jmeter/JMeterService.java
View File

@ -1,27 +1,25 @@
package io.metersphere.api.jmeter; package io.metersphere.api.jmeter;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import io.metersphere.api.dto.JvmInfoDTO;
import io.metersphere.api.dto.RunRequest; import io.metersphere.api.dto.RunRequest;
import io.metersphere.api.dto.automation.ExecuteType; import io.metersphere.api.dto.automation.ExecuteType;
import io.metersphere.api.dto.automation.RunModeConfig; import io.metersphere.api.dto.automation.RunModeConfig;
import io.metersphere.api.dto.definition.request.MsTestPlan;
import io.metersphere.api.service.ApiScenarioReportService; import io.metersphere.api.service.ApiScenarioReportService;
import io.metersphere.base.domain.TestResource;
import io.metersphere.base.domain.TestResourcePool; import io.metersphere.base.domain.TestResourcePool;
import io.metersphere.base.mapper.TestResourcePoolMapper; import io.metersphere.base.mapper.TestResourcePoolMapper;
import io.metersphere.commons.constants.ApiRunMode; import io.metersphere.commons.constants.ApiRunMode;
import io.metersphere.commons.constants.ResourcePoolTypeEnum; import io.metersphere.commons.constants.ResourcePoolTypeEnum;
import io.metersphere.commons.constants.TriggerMode; import io.metersphere.commons.constants.TriggerMode;
import io.metersphere.commons.exception.MSException; import io.metersphere.commons.exception.MSException;
import io.metersphere.commons.utils.*; import io.metersphere.commons.utils.CommonBeanFactory;
import io.metersphere.commons.utils.LogUtil;
import io.metersphere.config.JmeterProperties; import io.metersphere.config.JmeterProperties;
import io.metersphere.dto.BaseSystemConfigDTO; import io.metersphere.dto.BaseSystemConfigDTO;
import io.metersphere.dto.NodeDTO;
import io.metersphere.i18n.Translator; import io.metersphere.i18n.Translator;
import io.metersphere.performance.engine.Engine; import io.metersphere.performance.engine.Engine;
import io.metersphere.performance.engine.EngineFactory; import io.metersphere.performance.engine.EngineFactory;
import io.metersphere.service.SystemParameterService; import io.metersphere.service.SystemParameterService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPost;
@ -37,16 +35,16 @@ import org.apache.jmeter.util.JMeterUtils;
import org.apache.jmeter.visualizers.backend.BackendListener; import org.apache.jmeter.visualizers.backend.BackendListener;
import org.apache.jorphan.collections.HashTree; import org.apache.jorphan.collections.HashTree;
import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.http.ResponseEntity; import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.RestTemplate;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.*; import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.util.List;
@Service @Service
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
@ -55,11 +53,9 @@ public class JMeterService {
@Resource @Resource
private JmeterProperties jmeterProperties; private JmeterProperties jmeterProperties;
@Resource @Resource
ResourcePoolCalculation resourcePoolCalculation;
@Resource
private RestTemplate restTemplate;
@Resource
private TestResourcePoolMapper testResourcePoolMapper; private TestResourcePoolMapper testResourcePoolMapper;
@Resource
private KafkaTemplate<String, Object> kafkaTemplate;
@PostConstruct @PostConstruct
public void init() { public void init() {
@ -145,7 +141,7 @@ public class JMeterService {
} }
public void runTest(String testId, String reportId, String runMode, public void runTest(String testId, String reportId, String runMode,
String testPlanScenarioId, RunModeConfig config) { String testPlanScenarioId, RunModeConfig config, HashTree hashTree) {
// 获取可以执行的资源池 // 获取可以执行的资源池
String resourcePoolId = config.getResourcePoolId(); String resourcePoolId = config.getResourcePoolId();
BaseSystemConfigDTO baseInfo = config.getBaseInfo(); BaseSystemConfigDTO baseInfo = config.getBaseInfo();
@ -179,33 +175,8 @@ public class JMeterService {
MSException.throwException(e.getMessage()); MSException.throwException(e.getMessage());
} }
} else { } else {
TestResource testResource = null; runRequest.setJmx(new MsTestPlan().getJmx(hashTree));
List<JvmInfoDTO> testResources = config.getTestResources(); kafkaTemplate.send(MsKafkaListener.EXEC_TOPIC, JSON.toJSONString(runRequest));
if (CollectionUtils.isEmpty(testResources)) {
testResource = resourcePoolCalculation.getPool(resourcePoolId);
} else {
int index = (int) (Math.random() * testResources.size());
JvmInfoDTO jvmInfoDTO = testResources.get(index);
testResource = testResources.get(index).getTestResource();
}
String configuration = testResource.getConfiguration();
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
String nodeIp = node.getIp();
Integer port = node.getPort();
try {
String uri = String.format(BASE_URL + "/jmeter/api/start", nodeIp, port);
ResponseEntity<String> resultEntity = restTemplate.postForEntity(uri, runRequest, String.class);
String result = resultEntity.getBody(); // this.send(uri, runRequest);
if (StringUtils.isEmpty(result) || !StringUtils.equals("SUCCESS", result)) {
// 清理零时报告
ApiScenarioReportService apiScenarioReportService = CommonBeanFactory.getBean(ApiScenarioReportService.class);
apiScenarioReportService.delete(reportId);
MSException.throwException("执行失败：" + result);
}
} catch (Exception e) {
e.printStackTrace();
MSException.throwException(runRequest.getReportId() + "：" + e.getMessage());
}
} }
} }

2
backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java
View File

@ -16,6 +16,8 @@ import javax.annotation.Resource;
@Service @Service
public class MsKafkaListener { public class MsKafkaListener {
public static final String TOPICS = "ms-api-exec-topic"; public static final String TOPICS = "ms-api-exec-topic";
public final static String EXEC_TOPIC = "ms-automation-exec-topic";
public static final String CONSUME_ID = "ms-api-exec-consume"; public static final String CONSUME_ID = "ms-api-exec-consume";
@KafkaListener(id = CONSUME_ID, topics = TOPICS, groupId = "${spring.kafka.consumer.group-id}") @KafkaListener(id = CONSUME_ID, topics = TOPICS, groupId = "${spring.kafka.consumer.group-id}")

205
backend/src/main/java/io/metersphere/api/service/ApiAutomationService.java
View File

@ -25,7 +25,6 @@ import io.metersphere.api.jmeter.MessageCache;
import io.metersphere.api.jmeter.ReportCounter; import io.metersphere.api.jmeter.ReportCounter;
import io.metersphere.api.jmeter.ResourcePoolCalculation; import io.metersphere.api.jmeter.ResourcePoolCalculation;
import io.metersphere.api.parse.ApiImportParser; import io.metersphere.api.parse.ApiImportParser;
import io.metersphere.api.service.task.ParallelScenarioExecTask;
import io.metersphere.api.service.task.SerialScenarioExecTask; import io.metersphere.api.service.task.SerialScenarioExecTask;
import io.metersphere.base.domain.*; import io.metersphere.base.domain.*;
import io.metersphere.base.mapper.*; import io.metersphere.base.mapper.*;
@ -1052,7 +1051,11 @@ public class ApiAutomationService {
} }
try { try {
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) { if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
executeQueue.put(report.getId(), new RunModeDataDTO(item.getId(), report)); RunModeDataDTO runModeDataDTO = new RunModeDataDTO();
runModeDataDTO.setPlanEnvMap(planEnvMap);
runModeDataDTO.setReport(report);
runModeDataDTO.setScenario(item);
executeQueue.put(report.getId(), runModeDataDTO);
} else { } else {
// 生成报告和HashTree // 生成报告和HashTree
HashTree hashTree = generateHashTree(item, reportId, planEnvMap); HashTree hashTree = generateHashTree(item, reportId, planEnvMap);
@ -1088,108 +1091,130 @@ public class ApiAutomationService {
} }
} }
// 开始执行 // 开始执行
this.run(executeQueue, request, serialReportId); if (executeQueue != null && executeQueue.size() > 0) {
if (request.getConfig() != null && request.getConfig().getMode().equals(RunModeConstants.SERIAL.toString())) {
this.serial(executeQueue, request, serialReportId);
} else {
this.parallel(executeQueue, request);
}
}
return request.getId(); return request.getId();
} }
private void run(Map<String, RunModeDataDTO> executeQueue, RunScenarioRequest request, String serialReportId) { /**
// 开始选择执行模式 * 串行
if (executeQueue != null && executeQueue.size() > 0) { *
ExecutorService executorService = Executors.newFixedThreadPool(6); * @param executeQueue
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH); * @param request
ApiScenarioReportMapper batchMapper = sqlSession.getMapper(ApiScenarioReportMapper.class); * @param serialReportId
if (request.getConfig() != null && request.getConfig().getMode().equals(RunModeConstants.SERIAL.toString())) { */
// 非集合报告，先生成执行队列 private void serial(Map<String, RunModeDataDTO> executeQueue, RunScenarioRequest request, String serialReportId) {
if (StringUtils.isEmpty(serialReportId)) { ExecutorService executorService = Executors.newFixedThreadPool(executeQueue.size());
for (String reportId : executeQueue.keySet()) { SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
APIScenarioReportResult report = executeQueue.get(reportId).getReport(); ApiScenarioReportMapper batchMapper = sqlSession.getMapper(ApiScenarioReportMapper.class);
report.setStatus(APITestStatus.Waiting.name()); // 非集合报告，先生成执行队列
batchMapper.insert(report); if (StringUtils.isEmpty(serialReportId)) {
for (String reportId : executeQueue.keySet()) {
APIScenarioReportResult report = executeQueue.get(reportId).getReport();
report.setStatus(APITestStatus.Waiting.name());
batchMapper.insert(report);
}
sqlSession.flushStatements();
}
// 开始串行执行
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
List<String> reportIds = new LinkedList<>();
//记录串行执行中的环境参数，供下一个场景执行时使用。 <envId,<key,data>>
Map<String, Map<String, String>> execute_env_param_datas = new LinkedHashMap<>();
ApiTestEnvironmentService apiTestEnvironmentService = CommonBeanFactory.getBean(ApiTestEnvironmentService.class);
HashTreeUtil hashTreeUtil = new HashTreeUtil();
for (String key : executeQueue.keySet()) {
reportIds.add(key);
APIScenarioReportResult report = executeQueue.get(key).getReport();
if (StringUtils.isNotEmpty(serialReportId)) {
report.setExecuteType(ExecuteType.Marge.name());
apiScenarioReportMapper.insert(report);
} else {
report.setStatus(APITestStatus.Running.name());
report.setCreateTime(System.currentTimeMillis());
report.setUpdateTime(System.currentTimeMillis());
apiScenarioReportMapper.updateByPrimaryKey(report);
} }
sqlSession.flushStatements(); try {
} if (!execute_env_param_datas.isEmpty()) {
// 开始串行执行 HashTree hashTree = executeQueue.get(key).getHashTree();
Thread thread = new Thread(new Runnable() { hashTreeUtil.setEnvParamsMapToHashTree(hashTree, execute_env_param_datas);
@Override executeQueue.get(key).setHashTree(hashTree);
public void run() { }
List<String> reportIds = new LinkedList<>();
//记录串行执行中的环境参数，供下一个场景执行时使用。 <envId,<key,data>>
Map<String, Map<String, String>> execute_env_param_datas = new LinkedHashMap<>();
ApiTestEnvironmentService apiTestEnvironmentService = CommonBeanFactory.getBean(ApiTestEnvironmentService.class);
HashTreeUtil hashTreeUtil = new HashTreeUtil();
for (String key : executeQueue.keySet()) {
reportIds.add(key);
APIScenarioReportResult report = executeQueue.get(key).getReport();
if (StringUtils.isNotEmpty(serialReportId)) {
report.setExecuteType(ExecuteType.Marge.name());
apiScenarioReportMapper.insert(report);
} else {
report.setStatus(APITestStatus.Running.name());
report.setCreateTime(System.currentTimeMillis());
report.setUpdateTime(System.currentTimeMillis());
apiScenarioReportMapper.updateByPrimaryKey(report);
}
try {
if (!execute_env_param_datas.isEmpty()) {
try {
HashTree hashTree = executeQueue.get(key).getHashTree();
hashTreeUtil.setEnvParamsMapToHashTree(hashTree, execute_env_param_datas);
executeQueue.get(key).setHashTree(hashTree);
} catch (Exception e) {
}
}
Future<ApiScenarioReport> future = executorService.submit(new SerialScenarioExecTask(jMeterService, apiScenarioReportMapper, executeQueue.get(key), request));
ApiScenarioReport scenarioReport = future.get();
// 如果开启失败结束执行，则判断返回结果状态
if (request.getConfig().isOnSampleError()) {
if (scenarioReport == null || !scenarioReport.getStatus().equals("Success")) {
reportIds.remove(key);
break;
}
}
try { if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
Map<String, Map<String, String>> envParamsMap = hashTreeUtil.getEnvParamsDataByHashTree(executeQueue.get(key).getHashTree(), apiTestEnvironmentService); HashTree hashTree = generateHashTree(executeQueue.get(key).getScenario(), key, executeQueue.get(key).getPlanEnvMap());
execute_env_param_datas = hashTreeUtil.mergeParamDataMap(execute_env_param_datas, envParamsMap); executeQueue.get(key).setHashTree(hashTree);
} catch (Exception e) { }
} Future<ApiScenarioReport> future = executorService.submit(new SerialScenarioExecTask(jMeterService, apiScenarioReportMapper, executeQueue.get(key), request));
ApiScenarioReport scenarioReport = future.get();
} catch (Exception e) { // 如果开启失败结束执行，则判断返回结果状态
if (request.getConfig().isOnSampleError()) {
if (scenarioReport == null || !scenarioReport.getStatus().equals("Success")) {
reportIds.remove(key); reportIds.remove(key);
LogUtil.error("执行终止：" + e.getMessage());
break; break;
} }
} }
// 清理未执行的队列
if (reportIds.size() < executeQueue.size()) { Map<String, Map<String, String>> envParamsMap = hashTreeUtil.getEnvParamsDataByHashTree(executeQueue.get(key).getHashTree(), apiTestEnvironmentService);
List<String> removeList = executeQueue.entrySet().stream() execute_env_param_datas = hashTreeUtil.mergeParamDataMap(execute_env_param_datas, envParamsMap);
.filter(map -> !reportIds.contains(map.getKey())) } catch (Exception e) {
.map(map -> map.getKey()).collect(Collectors.toList()); reportIds.remove(key);
ApiScenarioReportExample example = new ApiScenarioReportExample(); LogUtil.error("执行终止：" + e.getMessage());
example.createCriteria().andIdIn(removeList); break;
apiScenarioReportMapper.deleteByExample(example);
}
// 更新集成报告
if (StringUtils.isNotEmpty(serialReportId)) {
apiScenarioReportService.margeReport(serialReportId, reportIds);
executeQueue.clear();
}
} }
});
thread.start();
} else {
// 开始并发执行
for (String reportId : executeQueue.keySet()) {
//存储报告
APIScenarioReportResult report = executeQueue.get(reportId).getReport();
batchMapper.insert(report);
} }
sqlSession.flushStatements(); // 清理未执行的队列
for (String reportId : executeQueue.keySet()) { if (reportIds.size() < executeQueue.size()) {
executorService.submit(new ParallelScenarioExecTask(jMeterService, executeQueue.get(reportId), request)); List<String> removeList = executeQueue.entrySet().stream().filter(map -> !reportIds.contains(map.getKey()))
.map(map -> map.getKey()).collect(Collectors.toList());
ApiScenarioReportExample example = new ApiScenarioReportExample();
example.createCriteria().andIdIn(removeList);
apiScenarioReportMapper.deleteByExample(example);
}
// 更新集成报告
if (StringUtils.isNotEmpty(serialReportId)) {
apiScenarioReportService.margeReport(serialReportId, reportIds);
executeQueue.clear();
} }
} }
});
thread.start();
}
/**
* 并行
*
* @param executeQueue
* @param request
*/
private void parallel(Map<String, RunModeDataDTO> executeQueue, RunScenarioRequest request) {
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
ApiScenarioReportMapper batchMapper = sqlSession.getMapper(ApiScenarioReportMapper.class);
// 开始并发执行
for (String reportId : executeQueue.keySet()) {
//存储报告
APIScenarioReportResult report = executeQueue.get(reportId).getReport();
batchMapper.insert(report);
} }
sqlSession.flushStatements();
for (String reportId : executeQueue.keySet()) {
if (request.getConfig() != null && StringUtils.isNotEmpty(request.getConfig().getResourcePoolId())) {
HashTree hashTree = generateHashTree(executeQueue.get(reportId).getScenario(), reportId, executeQueue.get(reportId).getPlanEnvMap());
jMeterService.runTest(executeQueue.get(reportId).getScenario().getId(), reportId, request.getRunMode(), request.getPlanScenarioId(), request.getConfig(), hashTree);
} else {
jMeterService.runLocal(reportId, executeQueue.get(reportId).getHashTree(),
TriggerMode.BATCH.name().equals(request.getTriggerMode()) ? TriggerMode.BATCH.name() : request.getReportId(), request.getRunMode());
}
}
executeQueue.clear();
} }
/** /**

2
backend/src/main/java/io/metersphere/api/service/ApiDefinitionService.java
View File

@ -689,7 +689,7 @@ public class ApiDefinitionService {
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) { if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
RunModeConfig configs = request.getConfig(); RunModeConfig configs = request.getConfig();
configs.setBaseInfo(CommonBeanFactory.getBean(SystemParameterService.class).getBaseInfo()); configs.setBaseInfo(CommonBeanFactory.getBean(SystemParameterService.class).getBaseInfo());
jMeterService.runTest(request.getId(), request.getId(), runMode, null, configs); jMeterService.runTest(request.getId(), request.getId(), runMode, null, configs, hashTree);
} else { } else {
jMeterService.runLocal(request.getId(), hashTree, request.getReportId(), runMode); jMeterService.runLocal(request.getId(), hashTree, request.getReportId(), runMode);
} }

135
backend/src/main/java/io/metersphere/api/service/ApiScenarioReportService.java
View File

@ -443,47 +443,50 @@ public class ApiScenarioReportService {
} }
public void margeReport(String reportId, List<String> reportIds) { public void margeReport(String reportId, List<String> reportIds) {
// 合并生成一份报告 ApiScenarioReport report = apiScenarioReportMapper.selectByPrimaryKey(reportId);
if (CollectionUtils.isNotEmpty(reportIds)) { // 要合并的报告已经被删除
TestResult testResult = new TestResult(); if (report == null) {
testResult.setTestId(UUID.randomUUID().toString()); MessageCache.cache.remove(reportId);
} else {
// 合并生成一份报告
if (CollectionUtils.isNotEmpty(reportIds)) {
TestResult testResult = new TestResult();
testResult.setTestId(UUID.randomUUID().toString());
StringBuilder idStr = new StringBuilder(); StringBuilder idStr = new StringBuilder();
reportIds.forEach(item -> { reportIds.forEach(item -> {
idStr.append("\"").append(item).append("\"").append(","); idStr.append("\"").append(item).append("\"").append(",");
}); });
List<ApiScenarioReportDetail> details = extApiScenarioReportDetailMapper.selectByIds(idStr.toString().substring(0, idStr.toString().length() - 1), "\"" + StringUtils.join(reportIds, ",") + "\""); List<ApiScenarioReportDetail> details = extApiScenarioReportDetailMapper.selectByIds(idStr.toString().substring(0, idStr.toString().length() - 1), "\"" + StringUtils.join(reportIds, ",") + "\"");
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
// 记录单场景通过率 // 记录单场景通过率
Map<String, String> passRateMap = new HashMap<>(); Map<String, String> passRateMap = new HashMap<>();
for (ApiScenarioReportDetail detail : details) { for (ApiScenarioReportDetail detail : details) {
try { try {
String content = new String(detail.getContent(), StandardCharsets.UTF_8); String content = new String(detail.getContent(), StandardCharsets.UTF_8);
TestResult scenarioResult = mapper.readValue(content, new TypeReference<TestResult>() { TestResult scenarioResult = mapper.readValue(content, new TypeReference<TestResult>() {
}); });
testResult.getScenarios().addAll(scenarioResult.getScenarios()); testResult.getScenarios().addAll(scenarioResult.getScenarios());
testResult.setTotal(testResult.getTotal() + scenarioResult.getTotal()); testResult.setTotal(testResult.getTotal() + scenarioResult.getTotal());
testResult.setError(testResult.getError() + scenarioResult.getError()); testResult.setError(testResult.getError() + scenarioResult.getError());
testResult.setPassAssertions(testResult.getPassAssertions() + scenarioResult.getPassAssertions()); testResult.setPassAssertions(testResult.getPassAssertions() + scenarioResult.getPassAssertions());
testResult.setSuccess(testResult.getSuccess() + scenarioResult.getSuccess()); testResult.setSuccess(testResult.getSuccess() + scenarioResult.getSuccess());
testResult.setTotalAssertions(scenarioResult.getTotalAssertions() + testResult.getTotalAssertions()); testResult.setTotalAssertions(scenarioResult.getTotalAssertions() + testResult.getTotalAssertions());
testResult.setScenarioTotal(testResult.getScenarioTotal() + scenarioResult.getScenarioTotal()); testResult.setScenarioTotal(testResult.getScenarioTotal() + scenarioResult.getScenarioTotal());
testResult.setScenarioSuccess(testResult.getScenarioSuccess() + scenarioResult.getScenarioSuccess()); testResult.setScenarioSuccess(testResult.getScenarioSuccess() + scenarioResult.getScenarioSuccess());
testResult.setScenarioError(testResult.getScenarioError() + scenarioResult.getScenarioError()); testResult.setScenarioError(testResult.getScenarioError() + scenarioResult.getScenarioError());
testResult.setConsole(scenarioResult.getConsole()); testResult.setConsole(scenarioResult.getConsole());
testResult.setScenarioStepError(scenarioResult.getScenarioStepError() + testResult.getScenarioStepError()); testResult.setScenarioStepError(scenarioResult.getScenarioStepError() + testResult.getScenarioStepError());
testResult.setScenarioStepSuccess(scenarioResult.getScenarioStepSuccess() + testResult.getScenarioStepSuccess()); testResult.setScenarioStepSuccess(scenarioResult.getScenarioStepSuccess() + testResult.getScenarioStepSuccess());
testResult.setScenarioStepTotal(scenarioResult.getScenarioStepTotal() + testResult.getScenarioStepTotal()); testResult.setScenarioStepTotal(scenarioResult.getScenarioStepTotal() + testResult.getScenarioStepTotal());
String passRate = new DecimalFormat("0%").format((float) scenarioResult.getSuccess() / (scenarioResult.getSuccess() + scenarioResult.getError())); String passRate = new DecimalFormat("0%").format((float) scenarioResult.getSuccess() / (scenarioResult.getSuccess() + scenarioResult.getError()));
passRateMap.put(detail.getReportId(), passRate); passRateMap.put(detail.getReportId(), passRate);
} catch (Exception e) { } catch (Exception e) {
LogUtil.error(e); LogUtil.error(e);
}
} }
}
ApiScenarioReport report = apiScenarioReportMapper.selectByPrimaryKey(reportId);
if (report != null) {
report.setExecuteType(ExecuteType.Saved.name()); report.setExecuteType(ExecuteType.Saved.name());
report.setStatus(testResult.getError() > 0 ? "Error" : "Success"); report.setStatus(testResult.getError() > 0 ? "Error" : "Success");
if (StringUtils.isNotEmpty(report.getTriggerMode()) && report.getTriggerMode().equals("CASE")) { if (StringUtils.isNotEmpty(report.getTriggerMode()) && report.getTriggerMode().equals("CASE")) {
@ -497,28 +500,28 @@ public class ApiScenarioReportService {
detail.setReportId(report.getId()); detail.setReportId(report.getId());
detail.setProjectId(report.getProjectId()); detail.setProjectId(report.getProjectId());
apiScenarioReportDetailMapper.insert(detail); apiScenarioReportDetailMapper.insert(detail);
// 更新场景状态
if (CollectionUtils.isNotEmpty(reportIds)) {
ApiScenarioReportExample scenarioReportExample = new ApiScenarioReportExample();
scenarioReportExample.createCriteria().andIdIn(reportIds);
List<ApiScenarioReport> reports = apiScenarioReportMapper.selectByExampleWithBLOBs(scenarioReportExample);
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
ApiScenarioMapper scenarioReportMapper = sqlSession.getMapper(ApiScenarioMapper.class);
reports.forEach(apiScenarioReport -> {
ApiScenarioWithBLOBs scenario = new ApiScenarioWithBLOBs();
scenario.setId(apiScenarioReport.getDescription());
scenario.setLastResult(StringUtils.equals("Error", apiScenarioReport.getStatus()) ? "Fail" : apiScenarioReport.getStatus());
scenario.setPassRate(passRateMap.get(apiScenarioReport.getId()));
scenario.setReportId(report.getId());
scenarioReportMapper.updateByPrimaryKeySelective(scenario);
});
sqlSession.flushStatements();
}
passRateMap.clear();
} }
// 更新场景状态
if (CollectionUtils.isNotEmpty(reportIds)) {
ApiScenarioReportExample scenarioReportExample = new ApiScenarioReportExample();
scenarioReportExample.createCriteria().andIdIn(reportIds);
List<ApiScenarioReport> reports = apiScenarioReportMapper.selectByExampleWithBLOBs(scenarioReportExample);
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
ApiScenarioMapper scenarioReportMapper = sqlSession.getMapper(ApiScenarioMapper.class);
reports.forEach(apiScenarioReport -> {
ApiScenarioWithBLOBs scenario = new ApiScenarioWithBLOBs();
scenario.setId(apiScenarioReport.getDescription());
scenario.setLastResult(StringUtils.equals("Error", apiScenarioReport.getStatus()) ? "Fail" : apiScenarioReport.getStatus());
scenario.setPassRate(passRateMap.get(apiScenarioReport.getId()));
scenario.setReportId(report.getId());
scenarioReportMapper.updateByPrimaryKeySelective(scenario);
});
sqlSession.flushStatements();
}
// 清理其他报告保留一份合并后的报告
passRateMap.clear();
deleteByIds(reportIds);
} }
// 清理其他报告保留一份合并后的报告
deleteByIds(reportIds);
} }
private void counter(TestResult result) { private void counter(TestResult result) {
@ -592,14 +595,14 @@ public class ApiScenarioReportService {
sendNotice(scenario); sendNotice(scenario);
} }
lastReport = report; lastReport = report;
} if (report.getExecuteType().equals(ExecuteType.Marge.name())) {
if (report.getExecuteType().equals(ExecuteType.Marge.name())) { Object obj = MessageCache.cache.get(report.getScenarioId());
Object obj = MessageCache.cache.get(report.getScenarioId()); if (obj != null) {
if (obj != null) { ReportCounter counter = (ReportCounter) obj;
ReportCounter counter = (ReportCounter) obj; counter.setNumber(counter.getNumber() + 1);
counter.setNumber(counter.getNumber() + 1); System.out.println("得到统计数量：" + counter.getNumber());
System.out.println("得到统计数量：" + counter.getNumber()); MessageCache.cache.put(report.getScenarioId(), counter);
MessageCache.cache.put(report.getScenarioId(), counter); }
} }
} }
} }

69
backend/src/main/java/io/metersphere/api/service/TestResultService.java
View File

@ -98,43 +98,44 @@ public class TestResultService {
testResult.setTestId(testId); testResult.setTestId(testId);
ApiScenarioReport scenarioReport = apiScenarioReportService.complete(testResult, runMode); ApiScenarioReport scenarioReport = apiScenarioReportService.complete(testResult, runMode);
//环境 //环境
ApiScenarioWithBLOBs apiScenario = apiAutomationService.getDto(scenarioReport.getScenarioId()); if (scenarioReport != null) {
String name = ""; ApiScenarioWithBLOBs apiScenario = apiAutomationService.getDto(scenarioReport.getScenarioId());
//执行人 String name = "";
String userName = ""; //执行人
//负责人 String userName = "";
String principal = ""; //负责人
if (apiScenario != null) { String principal = "";
String executionEnvironment = apiScenario.getScenarioDefinition(); if (apiScenario != null) {
JSONObject json = JSONObject.parseObject(executionEnvironment); String executionEnvironment = apiScenario.getScenarioDefinition();
if (json != null && json.getString("environmentMap") != null && json.getString("environmentMap").length() > 2) { JSONObject json = JSONObject.parseObject(executionEnvironment);
JSONObject environment = JSONObject.parseObject(json.getString("environmentMap")); if (json != null && json.getString("environmentMap") != null && json.getString("environmentMap").length() > 2) {
String environmentId = environment.get(apiScenario.getProjectId()).toString(); JSONObject environment = JSONObject.parseObject(json.getString("environmentMap"));
name = apiAutomationService.get(environmentId).getName(); String environmentId = environment.get(apiScenario.getProjectId()).toString();
name = apiAutomationService.get(environmentId).getName();
}
userName = apiAutomationService.getUser(apiScenario.getUserId());
principal = apiAutomationService.getUser(apiScenario.getPrincipal());
} }
userName = apiAutomationService.getUser(apiScenario.getUserId()); //报告内容
principal = apiAutomationService.getUser(apiScenario.getPrincipal()); reportTask = new ApiTestReportVariable();
} if (StringUtils.equalsAny(runMode, ApiRunMode.SCHEDULE_SCENARIO.name())) {
reportTask.setStatus(scenarioReport.getStatus());
//报告内容 reportTask.setId(scenarioReport.getId());
reportTask = new ApiTestReportVariable(); reportTask.setTriggerMode(scenarioReport.getTriggerMode());
if (StringUtils.equalsAny(runMode, ApiRunMode.SCHEDULE_SCENARIO.name())) { reportTask.setName(scenarioReport.getName());
reportTask.setStatus(scenarioReport.getStatus()); reportTask.setExecutor(userName);
reportTask.setId(scenarioReport.getId()); reportTask.setPrincipal(principal);
reportTask.setTriggerMode(scenarioReport.getTriggerMode()); reportTask.setExecutionTime(DateUtils.getTimeString(scenarioReport.getUpdateTime()));
reportTask.setName(scenarioReport.getName()); reportTask.setExecutionEnvironment(name);
reportTask.setExecutor(userName); SystemParameterService systemParameterService = CommonBeanFactory.getBean(SystemParameterService.class);
reportTask.setPrincipal(principal); assert systemParameterService != null;
reportTask.setExecutionTime(DateUtils.getTimeString(scenarioReport.getUpdateTime())); BaseSystemConfigDTO baseSystemConfigDTO = systemParameterService.getBaseInfo();
reportTask.setExecutionEnvironment(name); reportUrl = baseSystemConfigDTO.getUrl() + "/#/api/automation/report";
SystemParameterService systemParameterService = CommonBeanFactory.getBean(SystemParameterService.class);
assert systemParameterService != null;
BaseSystemConfigDTO baseSystemConfigDTO = systemParameterService.getBaseInfo();
reportUrl = baseSystemConfigDTO.getUrl() + "/#/api/automation/report";
}
testResult.setTestId(scenarioReport.getScenarioId());
planScenarioId = scenarioReport.getTestPlanScenarioId();
} }
testResult.setTestId(scenarioReport.getScenarioId());
planScenarioId = scenarioReport.getTestPlanScenarioId();
} else { } else {
apiTestService.changeStatus(testId, APITestStatus.Completed); apiTestService.changeStatus(testId, APITestStatus.Completed);
report = apiReportService.getRunningReport(testResult.getTestId()); report = apiReportService.getRunningReport(testResult.getTestId());

42
backend/src/main/java/io/metersphere/api/service/task/ParallelScenarioExecTask.java
View File

@ -1,42 +0,0 @@
/**
*
*/
package io.metersphere.api.service.task;
import io.metersphere.api.dto.RunModeDataDTO;
import io.metersphere.api.dto.automation.RunScenarioRequest;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.commons.constants.TriggerMode;
import io.metersphere.commons.exception.MSException;
import io.metersphere.commons.utils.LogUtil;
import org.apache.commons.lang3.StringUtils;
import java.util.concurrent.Callable;
public class ParallelScenarioExecTask<T> implements Callable<T> {
private RunScenarioRequest request;
private JMeterService jMeterService;
private RunModeDataDTO runModeDataDTO;
public ParallelScenarioExecTask(JMeterService jMeterService, RunModeDataDTO runModeDataDTO, RunScenarioRequest request) {
this.jMeterService = jMeterService;
this.request = request;
this.runModeDataDTO = runModeDataDTO;
}
@Override
public T call() {
try {
if (request.getConfig() != null && StringUtils.isNotEmpty(request.getConfig().getResourcePoolId())) {
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getReport().getId(), request.getRunMode(), request.getPlanScenarioId(), request.getConfig());
} else {
jMeterService.runLocal(runModeDataDTO.getReport().getId(), runModeDataDTO.getHashTree(), TriggerMode.BATCH.name().equals(request.getTriggerMode()) ? TriggerMode.BATCH.name() : request.getReportId(), request.getRunMode());
}
return null;
} catch (Exception ex) {
LogUtil.error(ex);
MSException.throwException(ex.getMessage());
return null;
}
}
}

2
backend/src/main/java/io/metersphere/api/service/task/SerialScenarioExecTask.java
View File

@ -34,7 +34,7 @@ public class SerialScenarioExecTask<T> implements Callable<T> {
public T call() { public T call() {
try { try {
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) { if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getReport().getId(), request.getRunMode(), request.getPlanScenarioId(), request.getConfig()); jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getReport().getId(), request.getRunMode(), request.getPlanScenarioId(), request.getConfig(), runModeDataDTO.getHashTree());
} else { } else {
jMeterService.runLocal(runModeDataDTO.getReport().getId(), runModeDataDTO.getHashTree(), TriggerMode.BATCH.name().equals(request.getTriggerMode()) ? TriggerMode.BATCH.name() : request.getReportId(), request.getRunMode()); jMeterService.runLocal(runModeDataDTO.getReport().getId(), runModeDataDTO.getHashTree(), TriggerMode.BATCH.name().equals(request.getTriggerMode()) ? TriggerMode.BATCH.name() : request.getReportId(), request.getRunMode());
} }

14
backend/src/main/java/io/metersphere/track/service/TestPlanApiCaseService.java
View File

@ -401,11 +401,12 @@ public class TestPlanApiCaseService {
mapper.updateByPrimaryKey(execResult); mapper.updateByPrimaryKey(execResult);
reportIds.add(execResult.getId()); reportIds.add(execResult.getId());
RunModeDataDTO modeDataDTO; RunModeDataDTO modeDataDTO;
// 生成报告和HashTree
HashTree hashTree = generateHashTree(testPlanApiCase.getId());
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) { if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
modeDataDTO = new RunModeDataDTO(testPlanApiCase.getId(), UUID.randomUUID().toString()); modeDataDTO = new RunModeDataDTO(testPlanApiCase.getId(), UUID.randomUUID().toString());
modeDataDTO.setHashTree(hashTree);
} else { } else {
// 生成报告和HashTree
HashTree hashTree = generateHashTree(testPlanApiCase.getId());
modeDataDTO = new RunModeDataDTO(hashTree, UUID.randomUUID().toString()); modeDataDTO = new RunModeDataDTO(hashTree, UUID.randomUUID().toString());
} }
modeDataDTO.setApiCaseId(execResult.getId()); modeDataDTO.setApiCaseId(execResult.getId());
@ -440,16 +441,21 @@ public class TestPlanApiCaseService {
// 开始并发执行 // 开始并发执行
for (TestPlanApiCase key : planApiCases) { for (TestPlanApiCase key : planApiCases) {
RunModeDataDTO modeDataDTO = null; RunModeDataDTO modeDataDTO = null;
// 生成报告和HashTree
HashTree hashTree = generateHashTree(key.getId());
if (StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) { if (StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
modeDataDTO = new RunModeDataDTO(key.getId(), UUID.randomUUID().toString()); modeDataDTO = new RunModeDataDTO(key.getId(), UUID.randomUUID().toString());
} else { } else {
// 生成报告和HashTree
HashTree hashTree = generateHashTree(key.getId());
modeDataDTO = new RunModeDataDTO(hashTree, UUID.randomUUID().toString()); modeDataDTO = new RunModeDataDTO(hashTree, UUID.randomUUID().toString());
} }
ApiDefinitionExecResult report = addResult(request, key, APITestStatus.Running.name(), batchMapper); ApiDefinitionExecResult report = addResult(request, key, APITestStatus.Running.name(), batchMapper);
modeDataDTO.setApiCaseId(report.getId()); modeDataDTO.setApiCaseId(report.getId());
executorService.submit(new ParallelApiExecTask(jMeterService, mapper, modeDataDTO, request.getConfig(), ApiRunMode.API_PLAN.name())); executorService.submit(new ParallelApiExecTask(jMeterService, mapper, modeDataDTO, request.getConfig(), ApiRunMode.API_PLAN.name()));
if (request.getConfig() != null && StringUtils.isNotEmpty(request.getConfig().getResourcePoolId())) {
jMeterService.runTest(modeDataDTO.getTestId(), modeDataDTO.getApiCaseId(), ApiRunMode.API_PLAN.name(), null, request.getConfig(), hashTree);
} else {
jMeterService.runLocal(modeDataDTO.getTestId(), hashTree, TriggerMode.BATCH.name() , ApiRunMode.API_PLAN.name());
}
} }
sqlSession.flushStatements(); sqlSession.flushStatements();
} }

2
backend/src/main/java/io/metersphere/track/service/task/ParallelApiExecTask.java
View File

@ -32,7 +32,7 @@ public class ParallelApiExecTask<T> implements Callable<T> {
public T call() { public T call() {
try { try {
if (config != null && StringUtils.isNotBlank(config.getResourcePoolId())) { if (config != null && StringUtils.isNotBlank(config.getResourcePoolId())) {
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getApiCaseId(), runMode, null, config); jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getApiCaseId(), runMode, null, config, runModeDataDTO.getHashTree());
} else { } else {
jMeterService.runLocal(runModeDataDTO.getApiCaseId(), runModeDataDTO.getHashTree(), runModeDataDTO.getReport() != null ? runModeDataDTO.getReport().getTriggerMode() : null, runMode); jMeterService.runLocal(runModeDataDTO.getApiCaseId(), runModeDataDTO.getHashTree(), runModeDataDTO.getReport() != null ? runModeDataDTO.getReport().getTriggerMode() : null, runMode);
} }

2
backend/src/main/java/io/metersphere/track/service/task/SerialApiExecTask.java
View File

@ -34,7 +34,7 @@ public class SerialApiExecTask<T> implements Callable<T> {
public T call() { public T call() {
try { try {
if (config != null && StringUtils.isNotBlank(config.getResourcePoolId())) { if (config != null && StringUtils.isNotBlank(config.getResourcePoolId())) {
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getApiCaseId(), runMode, null, config); jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getApiCaseId(), runMode, null, config, runModeDataDTO.getHashTree());
} else { } else {
jMeterService.runLocal(runModeDataDTO.getApiCaseId(), runModeDataDTO.getHashTree(), runModeDataDTO.getReport() != null ? runModeDataDTO.getReport().getTriggerMode() : null, runMode); jMeterService.runLocal(runModeDataDTO.getApiCaseId(), runModeDataDTO.getHashTree(), runModeDataDTO.getReport() != null ? runModeDataDTO.getReport().getTriggerMode() : null, runMode);
} }

3
backend/src/main/resources/application.properties
View File

@ -51,7 +51,7 @@ spring.flyway.baseline-version=0
spring.flyway.encoding=UTF-8 spring.flyway.encoding=UTF-8
spring.flyway.validate-on-migrate=false spring.flyway.validate-on-migrate=false
spring.kafka.listener.missing-topics-fatal=false spring.kafka.listener.missing-topics-fatal=false
spring.kafka.producer.properties.max.request.size=32428800
spring.messages.basename=i18n/messages spring.messages.basename=i18n/messages
# kafka # kafka
@ -76,7 +76,6 @@ kafka.ssl.keystore-type=JKS
kafka.ssl.protocol=TLS kafka.ssl.protocol=TLS
kafka.ssl.provider= kafka.ssl.provider=
kafka.ssl.truststore-type= kafka.ssl.truststore-type=
# jmeter # jmeter
jmeter.home=/opt/jmeter jmeter.home=/opt/jmeter