fix (接口自动化): 并发执行优化

This commit is contained in:
fit2-zhao 2021-08-30 17:28:28 +08:00 committed by fit2-zhao
parent 59b1faec6e
commit 4443a88dca
14 changed files with 266 additions and 283 deletions

View File

@ -1,10 +1,13 @@
package io.metersphere.api.dto;
import io.metersphere.api.dto.automation.APIScenarioReportResult;
import io.metersphere.base.domain.ApiScenarioWithBLOBs;
import lombok.Getter;
import lombok.Setter;
import org.apache.jorphan.collections.HashTree;
import java.util.Map;
@Getter
@Setter
public class RunModeDataDTO {
@ -17,6 +20,11 @@ public class RunModeDataDTO {
//
private String apiCaseId;
private ApiScenarioWithBLOBs scenario;
private Map<String, String> planEnvMap;
public RunModeDataDTO(){
}
public RunModeDataDTO(String testId, String apiCaseId) {
this.testId = testId;
this.apiCaseId = apiCaseId;
@ -36,4 +44,10 @@ public class RunModeDataDTO {
this.hashTree = hashTree;
this.report = report;
}
public RunModeDataDTO(HashTree hashTree, APIScenarioReportResult report, String testId) {
this.hashTree = hashTree;
this.report = report;
this.testId = testId;
}
}

View File

@ -7,6 +7,8 @@ import io.metersphere.api.service.MsResultService;
import io.metersphere.api.service.TestResultService;
import org.apache.commons.lang3.StringUtils;
import org.apache.jmeter.samplers.SampleResult;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.Comparator;
@ -18,6 +20,8 @@ import java.util.Map;
* 获取结果和数据库操作分离
* 减少占用的数据库连接
*/
@Service
@Transactional(rollbackFor = Exception.class)
public class APIBackendListenerHandler {
@Resource

View File

@ -1,27 +1,25 @@
package io.metersphere.api.jmeter;
import com.alibaba.fastjson.JSON;
import io.metersphere.api.dto.JvmInfoDTO;
import io.metersphere.api.dto.RunRequest;
import io.metersphere.api.dto.automation.ExecuteType;
import io.metersphere.api.dto.automation.RunModeConfig;
import io.metersphere.api.dto.definition.request.MsTestPlan;
import io.metersphere.api.service.ApiScenarioReportService;
import io.metersphere.base.domain.TestResource;
import io.metersphere.base.domain.TestResourcePool;
import io.metersphere.base.mapper.TestResourcePoolMapper;
import io.metersphere.commons.constants.ApiRunMode;
import io.metersphere.commons.constants.ResourcePoolTypeEnum;
import io.metersphere.commons.constants.TriggerMode;
import io.metersphere.commons.exception.MSException;
import io.metersphere.commons.utils.*;
import io.metersphere.commons.utils.CommonBeanFactory;
import io.metersphere.commons.utils.LogUtil;
import io.metersphere.config.JmeterProperties;
import io.metersphere.dto.BaseSystemConfigDTO;
import io.metersphere.dto.NodeDTO;
import io.metersphere.i18n.Translator;
import io.metersphere.performance.engine.Engine;
import io.metersphere.performance.engine.EngineFactory;
import io.metersphere.service.SystemParameterService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
@ -37,16 +35,16 @@ import org.apache.jmeter.util.JMeterUtils;
import org.apache.jmeter.visualizers.backend.BackendListener;
import org.apache.jorphan.collections.HashTree;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.http.ResponseEntity;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.RestTemplate;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.*;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Field;
import java.util.List;
@Service
@Transactional(rollbackFor = Exception.class)
@ -55,11 +53,9 @@ public class JMeterService {
@Resource
private JmeterProperties jmeterProperties;
@Resource
ResourcePoolCalculation resourcePoolCalculation;
@Resource
private RestTemplate restTemplate;
@Resource
private TestResourcePoolMapper testResourcePoolMapper;
@Resource
private KafkaTemplate<String, Object> kafkaTemplate;
@PostConstruct
public void init() {
@ -145,7 +141,7 @@ public class JMeterService {
}
public void runTest(String testId, String reportId, String runMode,
String testPlanScenarioId, RunModeConfig config) {
String testPlanScenarioId, RunModeConfig config, HashTree hashTree) {
// 获取可以执行的资源池
String resourcePoolId = config.getResourcePoolId();
BaseSystemConfigDTO baseInfo = config.getBaseInfo();
@ -179,33 +175,8 @@ public class JMeterService {
MSException.throwException(e.getMessage());
}
} else {
TestResource testResource = null;
List<JvmInfoDTO> testResources = config.getTestResources();
if (CollectionUtils.isEmpty(testResources)) {
testResource = resourcePoolCalculation.getPool(resourcePoolId);
} else {
int index = (int) (Math.random() * testResources.size());
JvmInfoDTO jvmInfoDTO = testResources.get(index);
testResource = testResources.get(index).getTestResource();
}
String configuration = testResource.getConfiguration();
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
String nodeIp = node.getIp();
Integer port = node.getPort();
try {
String uri = String.format(BASE_URL + "/jmeter/api/start", nodeIp, port);
ResponseEntity<String> resultEntity = restTemplate.postForEntity(uri, runRequest, String.class);
String result = resultEntity.getBody(); // this.send(uri, runRequest);
if (StringUtils.isEmpty(result) || !StringUtils.equals("SUCCESS", result)) {
// 清理零时报告
ApiScenarioReportService apiScenarioReportService = CommonBeanFactory.getBean(ApiScenarioReportService.class);
apiScenarioReportService.delete(reportId);
MSException.throwException("执行失败:" + result);
}
} catch (Exception e) {
e.printStackTrace();
MSException.throwException(runRequest.getReportId() + "" + e.getMessage());
}
runRequest.setJmx(new MsTestPlan().getJmx(hashTree));
kafkaTemplate.send(MsKafkaListener.EXEC_TOPIC, JSON.toJSONString(runRequest));
}
}

View File

@ -16,6 +16,8 @@ import javax.annotation.Resource;
@Service
public class MsKafkaListener {
public static final String TOPICS = "ms-api-exec-topic";
public final static String EXEC_TOPIC = "ms-automation-exec-topic";
public static final String CONSUME_ID = "ms-api-exec-consume";
@KafkaListener(id = CONSUME_ID, topics = TOPICS, groupId = "${spring.kafka.consumer.group-id}")

View File

@ -25,7 +25,6 @@ import io.metersphere.api.jmeter.MessageCache;
import io.metersphere.api.jmeter.ReportCounter;
import io.metersphere.api.jmeter.ResourcePoolCalculation;
import io.metersphere.api.parse.ApiImportParser;
import io.metersphere.api.service.task.ParallelScenarioExecTask;
import io.metersphere.api.service.task.SerialScenarioExecTask;
import io.metersphere.base.domain.*;
import io.metersphere.base.mapper.*;
@ -1052,7 +1051,11 @@ public class ApiAutomationService {
}
try {
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
executeQueue.put(report.getId(), new RunModeDataDTO(item.getId(), report));
RunModeDataDTO runModeDataDTO = new RunModeDataDTO();
runModeDataDTO.setPlanEnvMap(planEnvMap);
runModeDataDTO.setReport(report);
runModeDataDTO.setScenario(item);
executeQueue.put(report.getId(), runModeDataDTO);
} else {
// 生成报告和HashTree
HashTree hashTree = generateHashTree(item, reportId, planEnvMap);
@ -1088,17 +1091,27 @@ public class ApiAutomationService {
}
}
// 开始执行
this.run(executeQueue, request, serialReportId);
if (executeQueue != null && executeQueue.size() > 0) {
if (request.getConfig() != null && request.getConfig().getMode().equals(RunModeConstants.SERIAL.toString())) {
this.serial(executeQueue, request, serialReportId);
} else {
this.parallel(executeQueue, request);
}
}
return request.getId();
}
private void run(Map<String, RunModeDataDTO> executeQueue, RunScenarioRequest request, String serialReportId) {
// 开始选择执行模式
if (executeQueue != null && executeQueue.size() > 0) {
ExecutorService executorService = Executors.newFixedThreadPool(6);
/**
* 串行
*
* @param executeQueue
* @param request
* @param serialReportId
*/
private void serial(Map<String, RunModeDataDTO> executeQueue, RunScenarioRequest request, String serialReportId) {
ExecutorService executorService = Executors.newFixedThreadPool(executeQueue.size());
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
ApiScenarioReportMapper batchMapper = sqlSession.getMapper(ApiScenarioReportMapper.class);
if (request.getConfig() != null && request.getConfig().getMode().equals(RunModeConstants.SERIAL.toString())) {
// 非集合报告先生成执行队列
if (StringUtils.isEmpty(serialReportId)) {
for (String reportId : executeQueue.keySet()) {
@ -1131,12 +1144,14 @@ public class ApiAutomationService {
}
try {
if (!execute_env_param_datas.isEmpty()) {
try {
HashTree hashTree = executeQueue.get(key).getHashTree();
hashTreeUtil.setEnvParamsMapToHashTree(hashTree, execute_env_param_datas);
executeQueue.get(key).setHashTree(hashTree);
} catch (Exception e) {
}
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
HashTree hashTree = generateHashTree(executeQueue.get(key).getScenario(), key, executeQueue.get(key).getPlanEnvMap());
executeQueue.get(key).setHashTree(hashTree);
}
Future<ApiScenarioReport> future = executorService.submit(new SerialScenarioExecTask(jMeterService, apiScenarioReportMapper, executeQueue.get(key), request));
ApiScenarioReport scenarioReport = future.get();
@ -1148,12 +1163,8 @@ public class ApiAutomationService {
}
}
try {
Map<String, Map<String, String>> envParamsMap = hashTreeUtil.getEnvParamsDataByHashTree(executeQueue.get(key).getHashTree(), apiTestEnvironmentService);
execute_env_param_datas = hashTreeUtil.mergeParamDataMap(execute_env_param_datas, envParamsMap);
} catch (Exception e) {
}
} catch (Exception e) {
reportIds.remove(key);
LogUtil.error("执行终止:" + e.getMessage());
@ -1162,8 +1173,7 @@ public class ApiAutomationService {
}
// 清理未执行的队列
if (reportIds.size() < executeQueue.size()) {
List<String> removeList = executeQueue.entrySet().stream()
.filter(map -> !reportIds.contains(map.getKey()))
List<String> removeList = executeQueue.entrySet().stream().filter(map -> !reportIds.contains(map.getKey()))
.map(map -> map.getKey()).collect(Collectors.toList());
ApiScenarioReportExample example = new ApiScenarioReportExample();
example.createCriteria().andIdIn(removeList);
@ -1177,7 +1187,17 @@ public class ApiAutomationService {
}
});
thread.start();
} else {
}
/**
* 并行
*
* @param executeQueue
* @param request
*/
private void parallel(Map<String, RunModeDataDTO> executeQueue, RunScenarioRequest request) {
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
ApiScenarioReportMapper batchMapper = sqlSession.getMapper(ApiScenarioReportMapper.class);
// 开始并发执行
for (String reportId : executeQueue.keySet()) {
//存储报告
@ -1186,10 +1206,15 @@ public class ApiAutomationService {
}
sqlSession.flushStatements();
for (String reportId : executeQueue.keySet()) {
executorService.submit(new ParallelScenarioExecTask(jMeterService, executeQueue.get(reportId), request));
}
if (request.getConfig() != null && StringUtils.isNotEmpty(request.getConfig().getResourcePoolId())) {
HashTree hashTree = generateHashTree(executeQueue.get(reportId).getScenario(), reportId, executeQueue.get(reportId).getPlanEnvMap());
jMeterService.runTest(executeQueue.get(reportId).getScenario().getId(), reportId, request.getRunMode(), request.getPlanScenarioId(), request.getConfig(), hashTree);
} else {
jMeterService.runLocal(reportId, executeQueue.get(reportId).getHashTree(),
TriggerMode.BATCH.name().equals(request.getTriggerMode()) ? TriggerMode.BATCH.name() : request.getReportId(), request.getRunMode());
}
}
executeQueue.clear();
}
/**

View File

@ -689,7 +689,7 @@ public class ApiDefinitionService {
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
RunModeConfig configs = request.getConfig();
configs.setBaseInfo(CommonBeanFactory.getBean(SystemParameterService.class).getBaseInfo());
jMeterService.runTest(request.getId(), request.getId(), runMode, null, configs);
jMeterService.runTest(request.getId(), request.getId(), runMode, null, configs, hashTree);
} else {
jMeterService.runLocal(request.getId(), hashTree, request.getReportId(), runMode);
}

View File

@ -443,6 +443,11 @@ public class ApiScenarioReportService {
}
public void margeReport(String reportId, List<String> reportIds) {
ApiScenarioReport report = apiScenarioReportMapper.selectByPrimaryKey(reportId);
// 要合并的报告已经被删除
if (report == null) {
MessageCache.cache.remove(reportId);
} else {
// 合并生成一份报告
if (CollectionUtils.isNotEmpty(reportIds)) {
TestResult testResult = new TestResult();
@ -482,8 +487,6 @@ public class ApiScenarioReportService {
}
}
ApiScenarioReport report = apiScenarioReportMapper.selectByPrimaryKey(reportId);
if (report != null) {
report.setExecuteType(ExecuteType.Saved.name());
report.setStatus(testResult.getError() > 0 ? "Error" : "Success");
if (StringUtils.isNotEmpty(report.getTriggerMode()) && report.getTriggerMode().equals("CASE")) {
@ -497,7 +500,6 @@ public class ApiScenarioReportService {
detail.setReportId(report.getId());
detail.setProjectId(report.getProjectId());
apiScenarioReportDetailMapper.insert(detail);
}
// 更新场景状态
if (CollectionUtils.isNotEmpty(reportIds)) {
ApiScenarioReportExample scenarioReportExample = new ApiScenarioReportExample();
@ -515,11 +517,12 @@ public class ApiScenarioReportService {
});
sqlSession.flushStatements();
}
// 清理其他报告保留一份合并后的报告
passRateMap.clear();
deleteByIds(reportIds);
}
}
// 清理其他报告保留一份合并后的报告
deleteByIds(reportIds);
}
private void counter(TestResult result) {
if (CollectionUtils.isEmpty(result.getScenarios())) {
@ -592,7 +595,6 @@ public class ApiScenarioReportService {
sendNotice(scenario);
}
lastReport = report;
}
if (report.getExecuteType().equals(ExecuteType.Marge.name())) {
Object obj = MessageCache.cache.get(report.getScenarioId());
if (obj != null) {
@ -603,6 +605,7 @@ public class ApiScenarioReportService {
}
}
}
}
return lastReport;
}

View File

@ -98,6 +98,7 @@ public class TestResultService {
testResult.setTestId(testId);
ApiScenarioReport scenarioReport = apiScenarioReportService.complete(testResult, runMode);
//环境
if (scenarioReport != null) {
ApiScenarioWithBLOBs apiScenario = apiAutomationService.getDto(scenarioReport.getScenarioId());
String name = "";
//执行人
@ -115,7 +116,6 @@ public class TestResultService {
userName = apiAutomationService.getUser(apiScenario.getUserId());
principal = apiAutomationService.getUser(apiScenario.getPrincipal());
}
//报告内容
reportTask = new ApiTestReportVariable();
if (StringUtils.equalsAny(runMode, ApiRunMode.SCHEDULE_SCENARIO.name())) {
@ -135,6 +135,7 @@ public class TestResultService {
}
testResult.setTestId(scenarioReport.getScenarioId());
planScenarioId = scenarioReport.getTestPlanScenarioId();
}
} else {
apiTestService.changeStatus(testId, APITestStatus.Completed);
report = apiReportService.getRunningReport(testResult.getTestId());

View File

@ -1,42 +0,0 @@
/**
*
*/
package io.metersphere.api.service.task;
import io.metersphere.api.dto.RunModeDataDTO;
import io.metersphere.api.dto.automation.RunScenarioRequest;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.commons.constants.TriggerMode;
import io.metersphere.commons.exception.MSException;
import io.metersphere.commons.utils.LogUtil;
import org.apache.commons.lang3.StringUtils;
import java.util.concurrent.Callable;
public class ParallelScenarioExecTask<T> implements Callable<T> {
private RunScenarioRequest request;
private JMeterService jMeterService;
private RunModeDataDTO runModeDataDTO;
public ParallelScenarioExecTask(JMeterService jMeterService, RunModeDataDTO runModeDataDTO, RunScenarioRequest request) {
this.jMeterService = jMeterService;
this.request = request;
this.runModeDataDTO = runModeDataDTO;
}
@Override
public T call() {
try {
if (request.getConfig() != null && StringUtils.isNotEmpty(request.getConfig().getResourcePoolId())) {
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getReport().getId(), request.getRunMode(), request.getPlanScenarioId(), request.getConfig());
} else {
jMeterService.runLocal(runModeDataDTO.getReport().getId(), runModeDataDTO.getHashTree(), TriggerMode.BATCH.name().equals(request.getTriggerMode()) ? TriggerMode.BATCH.name() : request.getReportId(), request.getRunMode());
}
return null;
} catch (Exception ex) {
LogUtil.error(ex);
MSException.throwException(ex.getMessage());
return null;
}
}
}

View File

@ -34,7 +34,7 @@ public class SerialScenarioExecTask<T> implements Callable<T> {
public T call() {
try {
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getReport().getId(), request.getRunMode(), request.getPlanScenarioId(), request.getConfig());
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getReport().getId(), request.getRunMode(), request.getPlanScenarioId(), request.getConfig(), runModeDataDTO.getHashTree());
} else {
jMeterService.runLocal(runModeDataDTO.getReport().getId(), runModeDataDTO.getHashTree(), TriggerMode.BATCH.name().equals(request.getTriggerMode()) ? TriggerMode.BATCH.name() : request.getReportId(), request.getRunMode());
}

View File

@ -401,11 +401,12 @@ public class TestPlanApiCaseService {
mapper.updateByPrimaryKey(execResult);
reportIds.add(execResult.getId());
RunModeDataDTO modeDataDTO;
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
modeDataDTO = new RunModeDataDTO(testPlanApiCase.getId(), UUID.randomUUID().toString());
} else {
// 生成报告和HashTree
HashTree hashTree = generateHashTree(testPlanApiCase.getId());
if (request.getConfig() != null && StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
modeDataDTO = new RunModeDataDTO(testPlanApiCase.getId(), UUID.randomUUID().toString());
modeDataDTO.setHashTree(hashTree);
} else {
modeDataDTO = new RunModeDataDTO(hashTree, UUID.randomUUID().toString());
}
modeDataDTO.setApiCaseId(execResult.getId());
@ -440,16 +441,21 @@ public class TestPlanApiCaseService {
// 开始并发执行
for (TestPlanApiCase key : planApiCases) {
RunModeDataDTO modeDataDTO = null;
// 生成报告和HashTree
HashTree hashTree = generateHashTree(key.getId());
if (StringUtils.isNotBlank(request.getConfig().getResourcePoolId())) {
modeDataDTO = new RunModeDataDTO(key.getId(), UUID.randomUUID().toString());
} else {
// 生成报告和HashTree
HashTree hashTree = generateHashTree(key.getId());
modeDataDTO = new RunModeDataDTO(hashTree, UUID.randomUUID().toString());
}
ApiDefinitionExecResult report = addResult(request, key, APITestStatus.Running.name(), batchMapper);
modeDataDTO.setApiCaseId(report.getId());
executorService.submit(new ParallelApiExecTask(jMeterService, mapper, modeDataDTO, request.getConfig(), ApiRunMode.API_PLAN.name()));
if (request.getConfig() != null && StringUtils.isNotEmpty(request.getConfig().getResourcePoolId())) {
jMeterService.runTest(modeDataDTO.getTestId(), modeDataDTO.getApiCaseId(), ApiRunMode.API_PLAN.name(), null, request.getConfig(), hashTree);
} else {
jMeterService.runLocal(modeDataDTO.getTestId(), hashTree, TriggerMode.BATCH.name() , ApiRunMode.API_PLAN.name());
}
}
sqlSession.flushStatements();
}

View File

@ -32,7 +32,7 @@ public class ParallelApiExecTask<T> implements Callable<T> {
public T call() {
try {
if (config != null && StringUtils.isNotBlank(config.getResourcePoolId())) {
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getApiCaseId(), runMode, null, config);
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getApiCaseId(), runMode, null, config, runModeDataDTO.getHashTree());
} else {
jMeterService.runLocal(runModeDataDTO.getApiCaseId(), runModeDataDTO.getHashTree(), runModeDataDTO.getReport() != null ? runModeDataDTO.getReport().getTriggerMode() : null, runMode);
}

View File

@ -34,7 +34,7 @@ public class SerialApiExecTask<T> implements Callable<T> {
public T call() {
try {
if (config != null && StringUtils.isNotBlank(config.getResourcePoolId())) {
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getApiCaseId(), runMode, null, config);
jMeterService.runTest(runModeDataDTO.getTestId(), runModeDataDTO.getApiCaseId(), runMode, null, config, runModeDataDTO.getHashTree());
} else {
jMeterService.runLocal(runModeDataDTO.getApiCaseId(), runModeDataDTO.getHashTree(), runModeDataDTO.getReport() != null ? runModeDataDTO.getReport().getTriggerMode() : null, runMode);
}

View File

@ -51,7 +51,7 @@ spring.flyway.baseline-version=0
spring.flyway.encoding=UTF-8
spring.flyway.validate-on-migrate=false
spring.kafka.listener.missing-topics-fatal=false
spring.kafka.producer.properties.max.request.size=32428800
spring.messages.basename=i18n/messages
# kafka
@ -76,7 +76,6 @@ kafka.ssl.keystore-type=JKS
kafka.ssl.protocol=TLS
kafka.ssl.provider=
kafka.ssl.truststore-type=
# jmeter
jmeter.home=/opt/jmeter