test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(接口自动化): 重构场景串行方法，利用队列串行

This commit is contained in:
fit2-zhao 2021-05-06 11:26:20 +08:00 committed by fit2-zhao
parent 7727610c2e
commit 4ad15075aa
8 changed files with 236 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/main/java/io/metersphere/api/jmeter/APIBackendListenerClient.java
View File

@ -166,7 +166,6 @@ public class APIBackendListenerClient extends AbstractBackendListenerClient impl
} else {
scenarioResult = scenarios.get(scenarioId);
}
if (result.isSuccessful()) {
scenarioResult.addSuccess();
testResult.addSuccess();

152
backend/src/main/java/io/metersphere/api/service/ApiAutomationService.java
View File

@ -21,7 +21,8 @@ import io.metersphere.api.dto.definition.request.variable.ScenarioVariable;
import io.metersphere.api.dto.scenario.environment.EnvironmentConfig;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.api.parse.ApiImportParser;
import io.metersphere.api.service.task.ParallelExecTask;
import io.metersphere.api.service.task.ParallelScenarioExecTask;
import io.metersphere.api.service.task.SerialScenarioExecTask;
import io.metersphere.base.domain.*;
import io.metersphere.base.mapper.*;
import io.metersphere.base.mapper.ext.*;
@ -60,7 +61,9 @@ import java.io.FileInputStream;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
@Service
@Transactional(rollbackFor = Exception.class)
public class ApiAutomationService {
@ -107,6 +110,8 @@ public class ApiAutomationService {
private UserMapper userMapper;
@Resource
private SystemParameterService systemParameterService;
@Resource
private ApiScenarioReportService apiScenarioReportService;
public ApiScenarioWithBLOBs getDto(String id) {
return apiScenarioMapper.selectByPrimaryKey(id);
@ -263,7 +268,11 @@ public class ApiAutomationService {
scenario.setId(request.getId());
scenario.setName(request.getName());
scenario.setProjectId(request.getProjectId());
scenario.setTags(request.getTags());
if (StringUtils.equals(request.getTags(), "[]")) {
scenario.setTags("");
} else {
scenario.setTags(request.getTags());
}
scenario.setApiScenarioModuleId(request.getApiScenarioModuleId());
scenario.setModulePath(request.getModulePath());
scenario.setLevel(request.getLevel());
@ -717,13 +726,12 @@ public class ApiAutomationService {
}
/**
* 场景测试并行执行
* 这种方法性能有问题 2021/04/12
* 场景测试执行
*
* @param request
* @return
*/
public String parallelRun(RunScenarioRequest request) {
public String modeRun(RunScenarioRequest request) {
ServiceUtils.getSelectAllIds(request, request.getCondition(),
(query) -> extApiScenarioMapper.selectIdsByQuery((ApiScenarioRequest) query));
@ -731,7 +739,11 @@ public class ApiAutomationService {
//检查是否有正在执行中的情景
this.checkScenarioIsRunning(ids);
List<ApiScenarioWithBLOBs> apiScenarios = extApiScenarioMapper.selectIds(ids);
StringBuilder idStr = new StringBuilder();
ids.forEach(item -> {
idStr.append("\"").append(item).append("\"").append(",");
});
List<ApiScenarioWithBLOBs> apiScenarios = extApiScenarioMapper.selectByIds(idStr.toString().substring(0, idStr.toString().length() - 1), "\"" + StringUtils.join(ids, ",") + "\"");
// 只有一个场景且没有测试步骤，则提示
if (apiScenarios != null && apiScenarios.size() == 1 && (apiScenarios.get(0).getStepTotal() == null || apiScenarios.get(0).getStepTotal() == 0)) {
MSException.throwException((apiScenarios.get(0).getName() + "，" + Translator.get("automation_exec_info")));
@ -739,14 +751,19 @@ public class ApiAutomationService {
// 环境检查
this.checkEnv(request, apiScenarios);
if (request.getConfig() != null && request.getConfig().getMode().equals("serial")) {
if (StringUtils.isNotEmpty(request.getConfig().getReportName())) {
request.setExecuteType(ExecuteType.Completed.name());
}
}
if (StringUtils.isEmpty(request.getTriggerMode())) {
request.setTriggerMode(ReportTriggerMode.MANUAL.name());
}
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
ApiScenarioReportMapper batchMapper = sqlSession.getMapper(ApiScenarioReportMapper.class);
String reportId = request.getId();
Map<String, HashTree> map = new LinkedHashMap<>();
Map<APIScenarioReportResult, HashTree> map = new LinkedHashMap<>();
List<String> scenarioIds = new ArrayList<>();
StringBuilder scenarioNames = new StringBuilder();
// 按照场景执行
for (ApiScenarioWithBLOBs item : apiScenarios) {
if (item.getStepTotal() == null || item.getStepTotal() == 0) {
@ -780,26 +797,76 @@ public class ApiAutomationService {
} catch (Exception ex) {
MSException.throwException("解析运行步骤失败！场景名称：" + item.getName());
}
//存储报告
batchMapper.insert(report);
// 调用执行方法
//jMeterService.runDefinition(report.getId(), hashTree, request.getReportId(), request.getRunMode());
map.put(report.getId(), hashTree);
map.put(report, hashTree);
scenarioIds.add(item.getId());
scenarioNames.append(item.getName()).append(",");
// 重置报告ID
reportId = UUID.randomUUID().toString();
}
sqlSession.flushStatements();
// 开始执行
ExecutorService executorService = Executors.newFixedThreadPool(map.size());
for (String key : map.keySet()) {
// jMeterService.runDefinition(key, map.get(key), request.getReportId(), request.getRunMode());
executorService.submit(new ParallelExecTask(jMeterService, key, map.get(key), request));
// 生成集成报告
String serialReportId = null;
if (request.getConfig() != null && request.getConfig().getMode().equals("serial") && StringUtils.equals(request.getConfig().getReportType(), "setReport") && StringUtils.isNotEmpty(request.getConfig().getReportName())) {
request.getConfig().setReportId(UUID.randomUUID().toString());
APIScenarioReportResult report = createScenarioReport(request.getConfig().getReportId(), JSON.toJSONString(scenarioIds), scenarioNames.deleteCharAt(scenarioNames.toString().length() - 1).toString(), ReportTriggerMode.MANUAL.name(),
ExecuteType.Saved.name(), request.getProjectId(), request.getReportUserID(), request.getConfig());
report.setName(request.getConfig().getReportName());
apiScenarioReportMapper.insert(report);
serialReportId = report.getId();
}
// 开始执行
this.run(map, request, serialReportId);
return request.getId();
}
private void run(Map<APIScenarioReportResult, HashTree> map, RunScenarioRequest request, String serialReportId) {
// 开始选择执行模式
ExecutorService executorService = Executors.newFixedThreadPool(map.size());
if (request.getConfig() != null && request.getConfig().getMode().equals("serial")) {
// 开始串行执行
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
List<String> reportIds = new LinkedList<>();
for (APIScenarioReportResult key : map.keySet()) {
apiScenarioReportMapper.insert(key);
reportIds.add(key.getId());
try {
Future<ApiScenarioReport> future = executorService.submit(new SerialScenarioExecTask(jMeterService, apiScenarioReportMapper, key.getId(), map.get(key), request));
ApiScenarioReport report = future.get();
// 如果开启失败结束执行，则判断返回结果状态
if (request.getConfig().isOnSampleError()) {
if (report == null || !report.getStatus().equals("Success")) {
break;
}
}
} catch (Exception e) {
LogUtil.error("执行终止：" + e.getMessage());
break;
}
}
// 更新集成报告
if (StringUtils.isNotEmpty(serialReportId)) {
apiScenarioReportService.margeReport(serialReportId, reportIds);
map.clear();
}
}
});
thread.start();
} else {
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
ApiScenarioReportMapper batchMapper = sqlSession.getMapper(ApiScenarioReportMapper.class);
// 开始并发执行
for (APIScenarioReportResult report : map.keySet()) {
//存储报告
batchMapper.insert(report);
executorService.submit(new ParallelScenarioExecTask(jMeterService, report.getId(), map.get(report), request));
}
sqlSession.flushStatements();
}
}
/**
* 生成HashTree
*
@ -812,15 +879,8 @@ public class ApiAutomationService {
HashTree jmeterHashTree = new ListedHashTree();
MsTestPlan testPlan = new MsTestPlan();
testPlan.setHashTree(new LinkedList<>());
if (request.getConfig() != null && request.getConfig().getMode().equals("serial")) {
testPlan.setSerializeThreadgroups(true);
if (StringUtils.isNotEmpty(request.getConfig().getReportName())) {
request.setExecuteType(ExecuteType.Completed.name());
}
}
try {
boolean isFirst = true;
List<String> reportList = new ArrayList<>();
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
ApiScenarioReportMapper batchMapper = sqlSession.getMapper(ApiScenarioReportMapper.class);
for (ApiScenarioWithBLOBs item : apiScenarios) {
@ -884,7 +944,6 @@ public class ApiAutomationService {
report = createScenarioReport(group.getName(), item.getId(), item.getName(), request.getTriggerMode() == null ? ReportTriggerMode.MANUAL.name() : request.getTriggerMode(),
request.getExecuteType(), item.getProjectId(), request.getReportUserID(), request.getConfig());
}
reportList.add(report.getId());
batchMapper.insert(report);
reportIds.add(group.getName());
}
@ -893,13 +952,6 @@ public class ApiAutomationService {
isFirst = false;
}
testPlan.toHashTree(jmeterHashTree, testPlan.getHashTree(), new ParameterConfig());
// 生成集成报告
if (request.getConfig() != null && request.getConfig().getMode().equals("serial") && StringUtils.isNotEmpty(request.getConfig().getReportName())) {
request.getConfig().setReportId(UUID.randomUUID().toString());
APIScenarioReportResult report = createScenarioReport(request.getConfig().getReportId(), JSON.toJSONString(reportList), request.getConfig().getReportName(), ReportTriggerMode.MANUAL.name(),
ExecuteType.Saved.name(), request.getProjectId(), request.getReportUserID(), request.getConfig());
batchMapper.insert(report);
}
sqlSession.flushStatements();
} catch (Exception ex) {
MSException.throwException(ex.getMessage());
@ -966,7 +1018,7 @@ public class ApiAutomationService {
* @param request
* @return
*/
public String serialRun(RunScenarioRequest request) {
public String excute(RunScenarioRequest request) {
ServiceUtils.getSelectAllIds(request, request.getCondition(),
(query) -> extApiScenarioMapper.selectIdsByQuery((ApiScenarioRequest) query));
List<String> ids = request.getIds();
@ -1004,19 +1056,21 @@ public class ApiAutomationService {
}
public String run(RunScenarioRequest request) {
if (request.getConfig() != null && request.getConfig().getMode().equals("serial")) {
return this.serialRun(request);
if (request.getConfig() != null) {
if (request.getConfig().getMode().equals("parallel")) {
// 校验并发数量
int count = 50;
BaseSystemConfigDTO dto = systemParameterService.getBaseInfo();
if (StringUtils.isNotEmpty(dto.getConcurrency())) {
count = Integer.parseInt(dto.getConcurrency());
}
if (request.getIds().size() > count) {
MSException.throwException("并发数量过大，请重新选择！");
}
}
return this.modeRun(request);
} else {
// 校验并发数量
int count = 50;
BaseSystemConfigDTO dto = systemParameterService.getBaseInfo();
if (StringUtils.isNotEmpty(dto.getConcurrency())) {
count = Integer.parseInt(dto.getConcurrency());
}
if (request.getIds().size() > count) {
MSException.throwException("并发数量过大，请重新选择！");
}
return this.parallelRun(request);
return this.excute(request);
}
}

4
backend/src/main/java/io/metersphere/api/service/ApiDefinitionService.java
View File

@ -300,7 +300,7 @@ public class ApiDefinitionService {
if (StringUtils.isNotEmpty(request.getTags()) && !StringUtils.equals(request.getTags(), "[]")) {
test.setTags(request.getTags());
} else {
test.setTags(null);
test.setTags("");
}
this.setModule(test);
apiDefinitionMapper.updateByPrimaryKeySelective(test);
@ -348,7 +348,7 @@ public class ApiDefinitionService {
if (StringUtils.isNotEmpty(request.getTags()) && !StringUtils.equals(request.getTags(), "[]")) {
test.setTags(request.getTags());
} else {
test.setTags(null);
test.setTags("");
}
apiDefinitionMapper.insert(test);
return test;

60
backend/src/main/java/io/metersphere/api/service/ApiScenarioReportService.java
View File

@ -1,6 +1,9 @@
package io.metersphere.api.service;
import com.alibaba.fastjson.JSON;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.metersphere.api.dto.APIReportBatchRequest;
import io.metersphere.api.dto.DeleteAPIReportRequest;
import io.metersphere.api.dto.QueryAPIReportRequest;
@ -19,10 +22,7 @@ import io.metersphere.base.mapper.ext.ExtApiScenarioReportMapper;
import io.metersphere.commons.constants.ApiRunMode;
import io.metersphere.commons.constants.ReportTriggerMode;
import io.metersphere.commons.exception.MSException;
import io.metersphere.commons.utils.CommonBeanFactory;
import io.metersphere.commons.utils.DateUtils;
import io.metersphere.commons.utils.ServiceUtils;
import io.metersphere.commons.utils.SessionUtils;
import io.metersphere.commons.utils.*;
import io.metersphere.i18n.Translator;
import io.metersphere.track.service.TestPlanReportService;
import org.apache.commons.collections4.CollectionUtils;
@ -231,7 +231,7 @@ public class ApiScenarioReportService {
returnReport = report;
reportIds.add(report.getId());
}
margeReport(result, scenarioIds, scenarioNames, runMode, projectId, userId, reportIds);
// margeReport(result, scenarioIds, scenarioNames, runMode, projectId, userId, reportIds);
return returnReport;
}
@ -307,7 +307,7 @@ public class ApiScenarioReportService {
reportIds.add(report.getId());
}
// 合并报告
margeReport(result, scenarioIds, scenarioNames, runMode, projectId, userId, reportIds);
// margeReport(result, scenarioIds, scenarioNames, runMode, projectId, userId, reportIds);
TestPlanReportService testPlanReportService = CommonBeanFactory.getBean(TestPlanReportService.class);
testPlanReportService.updateReport(testPlanReportIdList, ApiRunMode.SCHEDULE_SCENARIO_PLAN.name(), ReportTriggerMode.SCHEDULE.name());
@ -374,6 +374,52 @@ public class ApiScenarioReportService {
}
}
public void margeReport(String reportId, List<String> reportIds) {
// 合并生成一份报告
if (CollectionUtils.isNotEmpty(reportIds)) {
TestResult testResult = new TestResult();
testResult.setTestId(UUID.randomUUID().toString());
ApiScenarioReportDetailExample example = new ApiScenarioReportDetailExample();
example.createCriteria().andReportIdIn(reportIds);
List<ApiScenarioReportDetail> details = apiScenarioReportDetailMapper.selectByExampleWithBLOBs(example);
ObjectMapper mapper = new ObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
for (ApiScenarioReportDetail detail : details) {
try {
String content = new String(detail.getContent(), StandardCharsets.UTF_8);
TestResult scenarioResult = mapper.readValue(content, new TypeReference<TestResult>() {
});
testResult.getScenarios().addAll(scenarioResult.getScenarios());
testResult.setTotal(testResult.getTotal() + scenarioResult.getTotal());
testResult.setError(testResult.getError() + scenarioResult.getError());
testResult.setPassAssertions(testResult.getPassAssertions() + scenarioResult.getPassAssertions());
testResult.setSuccess(testResult.getSuccess() + scenarioResult.getSuccess());
testResult.setTotalAssertions(scenarioResult.getTotalAssertions() + testResult.getTotalAssertions());
} catch (Exception e) {
LogUtil.error(e.getMessage());
}
}
ApiScenarioReport report = apiScenarioReportMapper.selectByPrimaryKey(reportId);
if (report != null) {
report.setExecuteType(ExecuteType.Saved.name());
report.setStatus(testResult.getError() > 0 ? "Error" : "Success");
apiScenarioReportMapper.updateByPrimaryKey(report);
ApiScenarioReportDetail detail = new ApiScenarioReportDetail();
detail.setContent(JSON.toJSONString(testResult).getBytes(StandardCharsets.UTF_8));
detail.setReportId(report.getId());
detail.setProjectId(report.getProjectId());
apiScenarioReportDetailMapper.insert(detail);
}
// 清理其他报告保留一份合并后的报告
this.deleteByIds(reportIds);
}
}
public ApiScenarioReport updateScenario(TestResult result, String runMode) {
ApiScenarioReport lastReport = null;
StringBuilder scenarioIds = new StringBuilder();
@ -420,7 +466,7 @@ public class ApiScenarioReportService {
reportIds.add(report.getId());
}
// 合并生成一份报告
margeReport(result, scenarioIds, scenarioNames, runMode, projectId, userId, reportIds);
// margeReport(result, scenarioIds, scenarioNames, runMode, projectId, userId, reportIds);
return lastReport;
}

4
backend/src/main/java/io/metersphere/api/service/ApiTestCaseService.java
View File

@ -280,7 +280,7 @@ public class ApiTestCaseService {
test.setUpdateTime(System.currentTimeMillis());
test.setDescription(request.getDescription());
if (StringUtils.equals("[]", request.getTags())) {
test.setTags(null);
test.setTags("");
} else {
test.setTags(request.getTags());
}
@ -310,7 +310,7 @@ public class ApiTestCaseService {
test.setDescription(request.getDescription());
test.setNum(getNextNum(request.getApiDefinitionId()));
if (StringUtils.equals("[]", request.getTags())) {
test.setTags(null);
test.setTags("");
} else {
test.setTags(request.getTags());
}

7
backend/src/main/java/io/metersphere/api/service/task/ParallelExecTask.java → backend/src/main/java/io/metersphere/api/service/task/ParallelScenarioExecTask.java
View File

@ -3,6 +3,7 @@
*/
package io.metersphere.api.service.task;
import com.alibaba.fastjson.JSON;
import io.metersphere.api.dto.automation.RunScenarioRequest;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.commons.exception.MSException;
@ -11,13 +12,13 @@ import org.apache.jorphan.collections.HashTree;
import java.util.concurrent.Callable;
public class ParallelExecTask<T> implements Callable<T> {
public class ParallelScenarioExecTask<T> implements Callable<T> {
private RunScenarioRequest request;
private JMeterService jMeterService;
private HashTree hashTree;
private String id;
public ParallelExecTask(JMeterService jMeterService, String id, HashTree hashTree, RunScenarioRequest request) {
public ParallelScenarioExecTask(JMeterService jMeterService, String id, HashTree hashTree, RunScenarioRequest request) {
this.jMeterService = jMeterService;
this.request = request;
this.hashTree = hashTree;
@ -27,7 +28,7 @@ public class ParallelExecTask<T> implements Callable<T> {
@Override
public T call() {
try {
jMeterService.runDefinition(id, hashTree, request.getReportId(), request.getRunMode());
jMeterService.runSerial(JSON.toJSONString(id), hashTree, request.getReportId(), request.getRunMode(), request.getConfig());
return null;
} catch (Exception ex) {
LogUtil.error(ex.getMessage());

55
backend/src/main/java/io/metersphere/api/service/task/SerialScenarioExecTask.java Normal file
View File

@ -0,0 +1,55 @@
/**
*
*/
package io.metersphere.api.service.task;
import com.alibaba.fastjson.JSON;
import io.metersphere.api.dto.automation.RunScenarioRequest;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.base.domain.ApiScenarioReport;
import io.metersphere.base.mapper.ApiScenarioReportMapper;
import io.metersphere.commons.constants.APITestStatus;
import io.metersphere.commons.exception.MSException;
import io.metersphere.commons.utils.LogUtil;
import org.apache.jorphan.collections.HashTree;
import java.util.concurrent.Callable;
public class SerialScenarioExecTask<T> implements Callable<T> {
private RunScenarioRequest request;
private JMeterService jMeterService;
private ApiScenarioReportMapper apiScenarioReportMapper;
private HashTree hashTree;
ApiScenarioReport report = null;
private String id;
public SerialScenarioExecTask(JMeterService jMeterService, ApiScenarioReportMapper apiScenarioReportMapper, String id, HashTree hashTree, RunScenarioRequest request) {
this.jMeterService = jMeterService;
this.apiScenarioReportMapper = apiScenarioReportMapper;
this.request = request;
this.hashTree = hashTree;
this.id = id;
}
@Override
public T call() {
try {
jMeterService.runSerial(JSON.toJSONString(id), hashTree, request.getReportId(), request.getRunMode(), request.getConfig());
// 轮询查看报告状态，最多200次，防止死循环
int index = 1;
while (index < 200) {
Thread.sleep(3000);
index++;
report = apiScenarioReportMapper.selectByPrimaryKey(id);
if (report != null && !report.getStatus().equals(APITestStatus.Running.name())) {
break;
}
}
return (T) report;
} catch (Exception ex) {
LogUtil.error(ex.getMessage());
MSException.throwException(ex.getMessage());
return null;
}
}
}

21
frontend/src/business/components/api/automation/scenario/ApiScenarioList.vue
View File

@ -154,9 +154,7 @@
</div>
</el-card>
<batch-edit ref="batchEdit" @batchEdit="batchEdit" :typeArr="typeArr" :value-arr="valueArr" :dialog-title="$t('test_track.case.batch_edit_case')">
</batch-edit>
<batch-edit ref="batchEdit" @batchEdit="batchEdit" :typeArr="typeArr" :value-arr="valueArr" :dialog-title="$t('test_track.case.batch_edit_case')"/>
<batch-move @refresh="search" @moveSave="moveSave" ref="testBatchMove"/>
<ms-run-mode @handleRunBatch="handleRunBatch" ref="runMode"/>
</div>
@ -593,12 +591,27 @@
this.$refs.runMode.open();
},
orderBySelectRows(rows) {
let selectIds = Array.from(rows).map(row => row.id);
let array = [];
for (let i in this.tableData) {
if (selectIds.indexOf(this.tableData[i].id) !== -1) {
array.push(this.tableData[i].id);
}
}
return array;
},
handleRunBatch(config) {
this.infoDb = false;
let url = "/api/automation/run/batch";
let run = {config: config};
run.id = getUUID();
this.buildBatchParam(run);
//按照列表排序
let ids = this.orderBySelectRows(this.selectRows);
run.ids = ids;
run.projectId = this.projectId;
run.condition = this.condition;
this.$post(url, run, response => {
let data = response.data;
this.runVisible = false;