test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(接口测试): 禁用本地调试时接口执行优化附件传输方法 

--story=1011099 --user=宋天阳
执行附件本地缓存，避免每次执行都下载文件性能开销比较大
https://www.tapd.cn/55049933/s/1332544
This commit is contained in:
song-tianyang 2023-02-09 14:58:59 +08:00 committed by 建国
parent b99963faa2
commit d44ca14e7b
10 changed files with 87 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
api-test/backend/src/main/java/io/metersphere/api/jmeter/JMeterService.java
View File

@ -5,6 +5,7 @@ import io.metersphere.api.dto.definition.request.ElementUtil;
import io.metersphere.api.dto.definition.request.MsTestPlan; import io.metersphere.api.dto.definition.request.MsTestPlan;
import io.metersphere.api.exec.engine.EngineFactory; import io.metersphere.api.exec.engine.EngineFactory;
import io.metersphere.api.exec.queue.ExecThreadPoolExecutor; import io.metersphere.api.exec.queue.ExecThreadPoolExecutor;
import io.metersphere.api.jmeter.utils.JmxFileUtil;
import io.metersphere.api.jmeter.utils.ServerConfig; import io.metersphere.api.jmeter.utils.ServerConfig;
import io.metersphere.api.jmeter.utils.SmoothWeighted; import io.metersphere.api.jmeter.utils.SmoothWeighted;
import io.metersphere.base.domain.TestResource; import io.metersphere.base.domain.TestResource;
@ -12,22 +13,17 @@ import io.metersphere.commons.config.KafkaConfig;
import io.metersphere.commons.constants.ApiRunMode; import io.metersphere.commons.constants.ApiRunMode;
import io.metersphere.commons.constants.ExtendedParameter; import io.metersphere.commons.constants.ExtendedParameter;
import io.metersphere.commons.exception.MSException; import io.metersphere.commons.exception.MSException;
import io.metersphere.commons.utils.FixedCapacityUtil; import io.metersphere.commons.utils.*;
import io.metersphere.commons.utils.GenerateHashTreeUtil;
import io.metersphere.commons.utils.HashTreeUtil;
import io.metersphere.commons.utils.JSON;
import io.metersphere.config.JmeterProperties; import io.metersphere.config.JmeterProperties;
import io.metersphere.constants.BackendListenerConstants; import io.metersphere.constants.BackendListenerConstants;
import io.metersphere.constants.RunModeConstants; import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.JmeterRunRequestDTO; import io.metersphere.dto.*;
import io.metersphere.dto.NodeDTO;
import io.metersphere.dto.PluginConfigDTO;
import io.metersphere.dto.RunModeConfigDTO;
import io.metersphere.engine.Engine; import io.metersphere.engine.Engine;
import io.metersphere.jmeter.JMeterBase; import io.metersphere.jmeter.JMeterBase;
import io.metersphere.jmeter.LocalRunner; import io.metersphere.jmeter.LocalRunner;
import io.metersphere.service.ApiPoolDebugService; import io.metersphere.service.ApiPoolDebugService;
import io.metersphere.service.PluginService; import io.metersphere.service.PluginService;
import io.metersphere.service.RedisTemplateService;
import io.metersphere.service.RemakeReportService; import io.metersphere.service.RemakeReportService;
import io.metersphere.utils.LoggerUtil; import io.metersphere.utils.LoggerUtil;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
@ -53,8 +49,6 @@ import java.util.List;
@Service @Service
public class JMeterService { public class JMeterService {
public static final String BASE_URL = "http://%s:%d"; public static final String BASE_URL = "http://%s:%d";
public static final String POOL = "POOL";
@Resource @Resource
private JmeterProperties jmeterProperties; private JmeterProperties jmeterProperties;
@Resource @Resource
@ -62,6 +56,8 @@ public class JMeterService {
@Resource @Resource
private RedisTemplate<String, Object> redisTemplate; private RedisTemplate<String, Object> redisTemplate;
@Resource @Resource
private RedisTemplateService redisTemplateService;
@Resource
private RemakeReportService remakeReportService; private RemakeReportService remakeReportService;
@Resource @Resource
private ExecThreadPoolExecutor execThreadPoolExecutor; private ExecThreadPoolExecutor execThreadPoolExecutor;
@ -165,16 +161,21 @@ public class JMeterService {
// 缓存调试脚本 // 缓存调试脚本
if (request.getHashTree() != null) { if (request.getHashTree() != null) {
ElementUtil.coverArguments(request.getHashTree()); ElementUtil.coverArguments(request.getHashTree());
String key = StringUtils.join(request.getReportId(), "-", request.getTestId()); //解析HashTree里的文件信息
redisTemplate.opsForValue().set(key, new MsTestPlan().getJmx(request.getHashTree())); List<AttachmentBodyFile> attachmentBodyFileList = ApiFileUtil.getExecuteFileForNode(request.getHashTree(), request.getReportId());
if (CollectionUtils.isNotEmpty(attachmentBodyFileList)) {
redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId()), JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList));
}
redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteScriptKey(request.getReportId(), request.getTestId()), new MsTestPlan().getJmx(request.getHashTree()));
} }
LoggerUtil.info("开始发送请求[ " + request.getTestId() + " ] 到K8S节点执行", request.getReportId()); LoggerUtil.info("开始发送请求[ " + request.getTestId() + " ] 到K8S节点执行", request.getReportId());
final Engine engine = EngineFactory.createApiEngine(request); final Engine engine = EngineFactory.createApiEngine(request);
engine.start(); engine.start();
} catch (Exception e) { } catch (Exception e) {
remakeReportService.testEnded(request, e.getMessage()); remakeReportService.testEnded(request, e.getMessage());
String key = StringUtils.join(request.getReportId(), "-", request.getTestId()); redisTemplateService.delete(JmxFileUtil.getExecuteScriptKey(request.getReportId(), request.getTestId()));
redisTemplate.delete(key); redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId()));
LoggerUtil.error("调用K8S执行请求[ " + request.getTestId() + " ]失败：", request.getReportId(), e); LoggerUtil.error("调用K8S执行请求[ " + request.getTestId() + " ]失败：", request.getReportId(), e);
} }
} else if ((MapUtils.isNotEmpty(request.getExtendedParameters()) } else if ((MapUtils.isNotEmpty(request.getExtendedParameters())
@ -193,16 +194,23 @@ public class JMeterService {
if (request.getHashTree() != null) { if (request.getHashTree() != null) {
// 过程变量处理 // 过程变量处理
ElementUtil.coverArguments(request.getHashTree()); ElementUtil.coverArguments(request.getHashTree());
String key = StringUtils.join(request.getReportId(), "-", request.getTestId());
redisTemplate.opsForValue().set(key, new MsTestPlan().getJmx(request.getHashTree()));
//解析HashTree里的文件信息
List<AttachmentBodyFile> attachmentBodyFileList = ApiFileUtil.getExecuteFileForNode(request.getHashTree(), request.getReportId());
if (CollectionUtils.isNotEmpty(attachmentBodyFileList)) {
redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId()), JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList));
}
redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteScriptKey(request.getReportId(), request.getTestId()), new MsTestPlan().getJmx(request.getHashTree()));
request.setHashTree(null); request.setHashTree(null);
} }
apiPoolDebugService.run(request, resources); apiPoolDebugService.run(request, resources);
} catch (Exception e) { } catch (Exception e) {
LoggerUtil.error(e); LoggerUtil.error(e);
remakeReportService.remake(request); remakeReportService.remake(request);
String key = StringUtils.join(request.getReportId(), "-", request.getTestId()); redisTemplateService.delete(JmxFileUtil.getExecuteScriptKey(request.getReportId(), request.getTestId()));
redisTemplate.delete(key); redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId()));
LoggerUtil.error("发送请求[ " + request.getTestId() + " ] 执行失败,进行数据回滚：", request.getReportId(), e); LoggerUtil.error("发送请求[ " + request.getTestId() + " ] 执行失败,进行数据回滚：", request.getReportId(), e);
MSException.throwException("调用资源池执行失败，请检查资源池是否配置正常"); MSException.throwException("调用资源池执行失败，请检查资源池是否配置正常");
} }

8
api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java
View File

@ -8,6 +8,7 @@ import io.metersphere.commons.constants.ExtendedParameter;
import io.metersphere.commons.utils.JSON; import io.metersphere.commons.utils.JSON;
import io.metersphere.dto.ResultDTO; import io.metersphere.dto.ResultDTO;
import io.metersphere.service.ApiExecutionQueueService; import io.metersphere.service.ApiExecutionQueueService;
import io.metersphere.service.RedisTemplateService;
import io.metersphere.service.TestResultService; import io.metersphere.service.TestResultService;
import io.metersphere.utils.LoggerUtil; import io.metersphere.utils.LoggerUtil;
import lombok.Data; import lombok.Data;
@ -15,7 +16,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.data.redis.core.RedisTemplate;
import java.util.*; import java.util.*;
@ -25,7 +25,7 @@ public class KafkaListenerTask implements Runnable {
private ApiExecutionQueueService apiExecutionQueueService; private ApiExecutionQueueService apiExecutionQueueService;
private TestResultService testResultService; private TestResultService testResultService;
private RedisTemplate<String, Object> redisTemplate; private RedisTemplateService redisTemplateService;
private static final Map<String, String> RUN_MODE_MAP = new HashMap<String, String>() {{ private static final Map<String, String> RUN_MODE_MAP = new HashMap<String, String>() {{
this.put(ApiRunMode.SCHEDULE_API_PLAN.name(), "schedule-task"); this.put(ApiRunMode.SCHEDULE_API_PLAN.name(), "schedule-task");
@ -58,9 +58,7 @@ public class KafkaListenerTask implements Runnable {
if (dto == null) { if (dto == null) {
return; return;
} }
if (redisTemplate != null) { redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(dto.getReportId()));
redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId());
}
if (dto.getArbitraryData() != null && dto.getArbitraryData().containsKey(ExtendedParameter.TEST_END) if (dto.getArbitraryData() != null && dto.getArbitraryData().containsKey(ExtendedParameter.TEST_END)
&& (Boolean) dto.getArbitraryData().get(ExtendedParameter.TEST_END)) { && (Boolean) dto.getArbitraryData().get(ExtendedParameter.TEST_END)) {

12
api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java
View File

@ -12,6 +12,7 @@ import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.ResultDTO; import io.metersphere.dto.ResultDTO;
import io.metersphere.jmeter.JMeterBase; import io.metersphere.jmeter.JMeterBase;
import io.metersphere.service.ApiExecutionQueueService; import io.metersphere.service.ApiExecutionQueueService;
import io.metersphere.service.RedisTemplateService;
import io.metersphere.service.TestResultService; import io.metersphere.service.TestResultService;
import io.metersphere.utils.LoggerUtil; import io.metersphere.utils.LoggerUtil;
import io.metersphere.utils.RetryResultUtil; import io.metersphere.utils.RetryResultUtil;
@ -21,7 +22,6 @@ import org.apache.jmeter.samplers.SampleResult;
import org.apache.jmeter.services.FileServer; import org.apache.jmeter.services.FileServer;
import org.apache.jmeter.visualizers.backend.AbstractBackendListenerClient; import org.apache.jmeter.visualizers.backend.AbstractBackendListenerClient;
import org.apache.jmeter.visualizers.backend.BackendListenerContext; import org.apache.jmeter.visualizers.backend.BackendListenerContext;
import org.springframework.data.redis.core.RedisTemplate;
import java.io.Serializable; import java.io.Serializable;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@ -38,7 +38,7 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen
private ResultVO resultVO; private ResultVO resultVO;
@Resource @Resource
private RedisTemplate<String, Object> redisTemplate; private RedisTemplateService redisTemplateService;
/** /**
* 参数初始化方法 * 参数初始化方法
@ -64,9 +64,7 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen
if (dto.isRetryEnable()) { if (dto.isRetryEnable()) {
queues.addAll(sampleResults); queues.addAll(sampleResults);
} else { } else {
if (redisTemplate != null) { redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(dto.getReportId()));
redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId());
}
if (!StringUtils.equals(dto.getReportType(), RunModeConstants.SET_REPORT.toString())) { if (!StringUtils.equals(dto.getReportType(), RunModeConstants.SET_REPORT.toString())) {
dto.setConsole(FixedCapacityUtil.getJmeterLogger(getReportId(), false)); dto.setConsole(FixedCapacityUtil.getJmeterLogger(getReportId(), false));
@ -85,9 +83,7 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen
try { try {
LoggerUtil.info("进入TEST-END处理报告" + dto.getRunMode(), dto.getReportId()); LoggerUtil.info("进入TEST-END处理报告" + dto.getRunMode(), dto.getReportId());
if (redisTemplate != null) { redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(dto.getReportId()));
redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId());
}
super.teardownTest(context); super.teardownTest(context);
// 获取执行日志 // 获取执行日志

6
api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java
View File

@ -5,6 +5,7 @@ import io.metersphere.commons.constants.ApiRunMode;
import io.metersphere.commons.constants.KafkaTopicConstants; import io.metersphere.commons.constants.KafkaTopicConstants;
import io.metersphere.commons.utils.*; import io.metersphere.commons.utils.*;
import io.metersphere.service.ApiExecutionQueueService; import io.metersphere.service.ApiExecutionQueueService;
import io.metersphere.service.RedisTemplateService;
import io.metersphere.service.TestResultService; import io.metersphere.service.TestResultService;
import io.metersphere.service.definition.ApiDefinitionEnvService; import io.metersphere.service.definition.ApiDefinitionEnvService;
import io.metersphere.utils.LoggerUtil; import io.metersphere.utils.LoggerUtil;
@ -13,7 +14,6 @@ import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.Acknowledgment;
@ -40,7 +40,7 @@ public class MsKafkaListener {
private final static int WORK_QUEUE_SIZE = 10000; private final static int WORK_QUEUE_SIZE = 10000;
@Resource @Resource
private RedisTemplate<String, Object> redisTemplate; private RedisTemplateService redisTemplateService;
private final ThreadPoolExecutor threadPool = new ThreadPoolExecutor( private final ThreadPoolExecutor threadPool = new ThreadPoolExecutor(
CORE_POOL_SIZE, CORE_POOL_SIZE,
@ -59,7 +59,7 @@ public class MsKafkaListener {
task.setApiExecutionQueueService(apiExecutionQueueService); task.setApiExecutionQueueService(apiExecutionQueueService);
task.setTestResultService(testResultService); task.setTestResultService(testResultService);
task.setRecord(item); task.setRecord(item);
task.setRedisTemplate(redisTemplate); task.setRedisTemplateService(redisTemplateService);
threadPool.execute(task); threadPool.execute(task);
}); });
JvmUtil.memoryInfo(); JvmUtil.memoryInfo();

10
api-test/backend/src/main/java/io/metersphere/api/jmeter/utils/JmxFileUtil.java
View File

@ -12,7 +12,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
public class JmxFileUtil { public class JmxFileUtil {
public static final String REDIS_JMX_FILE_PREFIX = "JMX.FILE."; public static final String REDIS_JMX_EXECUTE_FILE_PREFIX = "JMX.FILE.";
private static final String JMX_INFO_KEY_ID = "id"; private static final String JMX_INFO_KEY_ID = "id";
private static final String JMX_INFO_KEY_FILE_PATH = "filePath"; private static final String JMX_INFO_KEY_FILE_PATH = "filePath";
@ -21,6 +21,14 @@ public class JmxFileUtil {
private static final String JMX_INFO_KEY_FILE_STORAGE = "storage"; private static final String JMX_INFO_KEY_FILE_STORAGE = "storage";
public static String getExecuteFileKeyInRedis(String reportId) {
return StringUtils.join(REDIS_JMX_EXECUTE_FILE_PREFIX, reportId);
}
public static String getExecuteScriptKey(String reportId, String testId) {
return StringUtils.join(reportId, "-", testId);
}
public static String getRedisJmxFileString(List<AttachmentBodyFile> listFile) { public static String getRedisJmxFileString(List<AttachmentBodyFile> listFile) {
List<Map<String, String>> jmxFileList = new ArrayList<>(); List<Map<String, String>> jmxFileList = new ArrayList<>();
listFile.forEach(file -> { listFile.forEach(file -> {

79
api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java
View File

@ -23,6 +23,7 @@ import org.apache.jorphan.collections.HashTree;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import java.io.File; import java.io.File;
import java.util.ArrayList;
import java.util.List; import java.util.List;
public class ApiFileUtil extends FileUtils { public class ApiFileUtil extends FileUtils {
@ -110,7 +111,13 @@ public class ApiFileUtil extends FileUtils {
} }
} }
public static void formatFilePathForNode(HashTree tree, String reportId, List<AttachmentBodyFile> fileList) { public static List<AttachmentBodyFile> getExecuteFileForNode(HashTree tree, String reportId) {
List<AttachmentBodyFile> fileList = new ArrayList<>();
formatFilePathForNode(tree, reportId, fileList);
return fileList;
}
private static void formatFilePathForNode(HashTree tree, String reportId, List<AttachmentBodyFile> fileList) {
if (tree != null) { if (tree != null) {
if (fileMetadataService == null) { if (fileMetadataService == null) {
fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class); fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class);
@ -217,76 +224,6 @@ public class ApiFileUtil extends FileUtils {
return StringUtils.join(BODY_FILE_DIR, File.separator, reportId, File.separator, fileName); return StringUtils.join(BODY_FILE_DIR, File.separator, reportId, File.separator, fileName);
} }
// public static void formatFilePathForNode(HashTree tree, String reportId, List<AttachmentBodyFile> fileList) {
// if (fileMetadataService == null) {
// fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class);
// }
// for (Object key : tree.keySet()) {
// if (key == null) {
// continue;
// }
// HashTree node = tree.get(key);
// if (key instanceof HTTPSamplerProxy) {
// formatHttpFilePathForNode(key, reportId, fileList);
// } else if (key instanceof CSVDataSet) {
// formatCsvFilePathForNode(key, reportId, fileList);
// }
// if (node != null) {
// formatFilePathForNode(node, reportId, fileList);
// }
// }
// }
// private static void formatCsvFilePathForNode(Object key, String reportId) {
// CSVDataSet source = (CSVDataSet) key;
// if (StringUtils.isNotEmpty(source.getPropertyAsString(ElementConstants.FILENAME))) {
// if (source.getPropertyAsBoolean(ElementConstants.IS_REF)) {
// FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById(
// source.getPropertyAsString(ElementConstants.FILE_ID));
// if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) {
// String path = getFilePathInJxm(reportId, fileMetadata.getName());
// ((CSVDataSet) key).setProperty(ElementConstants.FILENAME, path);
// ((CSVDataSet) key).setProperty(ElementConstants.FILE_STORAGE, fileMetadata.getStorage());
// ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_NAME, fileMetadata.getName());
// ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_UPDATE_TIME, fileMetadata.getUpdateTime());
// ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_PROJECT_ID, fileMetadata.getProjectId());
// if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) {
// ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_ATTACH_INFO, fileMetadata.getAttachInfo());
// }
// }
// }
// }
// }
// private static void formatHttpFilePathForNode(Object key, String reportId, List<AttachmentBodyFile> fileList) {
// if (key == null) {
// return;
// }
// HTTPSamplerProxy source = (HTTPSamplerProxy) key;
// for (HTTPFileArg httpFileArg : source.getHTTPFiles()) {
// if (httpFileArg.getPropertyAsBoolean(ElementConstants.IS_REF)) {
// FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById(
// httpFileArg.getPropertyAsString(ElementConstants.FILE_ID));
// if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) {
// String path = getFilePathInJxm(reportId, fileMetadata.getName());
// httpFileArg.setPath(path);
// httpFileArg.setProperty(ElementConstants.FILE_STORAGE, fileMetadata.getStorage());
// httpFileArg.setProperty(ElementConstants.REF_FILE_NAME, fileMetadata.getName());
// httpFileArg.setProperty(ElementConstants.REF_FILE_UPDATE_TIME, fileMetadata.getUpdateTime());
// httpFileArg.setProperty(ElementConstants.REF_FILE_PROJECT_ID, fileMetadata.getProjectId());
// if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) {
// httpFileArg.setProperty(ElementConstants.REF_FILE_ATTACH_INFO, fileMetadata.getAttachInfo());
// }
//
// fileList.add(new AttachmentBodyFile() {{
// this.setFileMetadataId(fileMetadata.getId());
// }});
// }
// }
// }
//
//
/** /**
* 获取当前jmx 涉及到的文件 执行时 * 获取当前jmx 涉及到的文件 执行时
* *

8
api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java
View File

@ -2,10 +2,10 @@ package io.metersphere.controller;
import io.metersphere.api.dto.BodyFileRequest; import io.metersphere.api.dto.BodyFileRequest;
import io.metersphere.api.jmeter.JMeterThreadUtils; import io.metersphere.api.jmeter.JMeterThreadUtils;
import io.metersphere.api.jmeter.utils.JmxFileUtil;
import io.metersphere.service.ApiJMeterFileService; import io.metersphere.service.ApiJMeterFileService;
import io.metersphere.utils.LoggerUtil; import io.metersphere.utils.LoggerUtil;
import jakarta.annotation.Resource; import jakarta.annotation.Resource;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
@ -68,8 +68,8 @@ public class ApiJMeterFileController {
@PostMapping("download/files") @PostMapping("download/files")
public ResponseEntity<byte[]> downloadJmeterFiles(@RequestBody BodyFileRequest request) { public ResponseEntity<byte[]> downloadJmeterLocalFiles(@RequestBody BodyFileRequest request) {
byte[] bytes = apiJmeterFileService.zipFilesToByteArray(request); byte[] bytes = apiJmeterFileService.zipLocalFilesToByteArray(request);
return ResponseEntity.ok() return ResponseEntity.ok()
.contentType(MediaType.parseMediaType("application/octet-stream")) .contentType(MediaType.parseMediaType("application/octet-stream"))
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + request.getReportId() + ".zip\"") .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + request.getReportId() + ".zip\"")
@ -78,7 +78,7 @@ public class ApiJMeterFileController {
@GetMapping("get-script") @GetMapping("get-script")
public String getScript(@RequestParam("reportId") String reportId, @RequestParam("testId") String testId) { public String getScript(@RequestParam("reportId") String reportId, @RequestParam("testId") String testId) {
String key = StringUtils.join(reportId, "-", testId); String key = JmxFileUtil.getExecuteScriptKey(reportId, testId);
LoggerUtil.info("获取执行脚本", key); LoggerUtil.info("获取执行脚本", key);
Object script = redisTemplate.opsForValue().get(key); Object script = redisTemplate.opsForValue().get(key);
redisTemplate.delete(key); redisTemplate.delete(key);

6
api-test/backend/src/main/java/io/metersphere/service/ApiExecutionQueueService.java
View File

@ -6,6 +6,7 @@ import io.metersphere.api.exec.queue.DBTestQueue;
import io.metersphere.api.exec.scenario.ApiScenarioSerialService; import io.metersphere.api.exec.scenario.ApiScenarioSerialService;
import io.metersphere.api.jmeter.JMeterService; import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.api.jmeter.JMeterThreadUtils; import io.metersphere.api.jmeter.JMeterThreadUtils;
import io.metersphere.api.jmeter.utils.JmxFileUtil;
import io.metersphere.base.domain.*; import io.metersphere.base.domain.*;
import io.metersphere.base.mapper.*; import io.metersphere.base.mapper.*;
import io.metersphere.base.mapper.ext.BaseApiExecutionQueueMapper; import io.metersphere.base.mapper.ext.BaseApiExecutionQueueMapper;
@ -23,6 +24,7 @@ import io.metersphere.dto.ResultDTO;
import io.metersphere.dto.RunModeConfigDTO; import io.metersphere.dto.RunModeConfigDTO;
import io.metersphere.service.scenario.ApiScenarioReportService; import io.metersphere.service.scenario.ApiScenarioReportService;
import io.metersphere.utils.LoggerUtil; import io.metersphere.utils.LoggerUtil;
import jakarta.annotation.Resource;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.BooleanUtils;
@ -32,7 +34,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import jakarta.annotation.Resource;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -377,6 +378,7 @@ public class ApiExecutionQueueService {
dto.setQueueId(item.getQueueId()); dto.setQueueId(item.getQueueId());
dto.setTestId(item.getTestId()); dto.setTestId(item.getTestId());
if (StringUtils.equalsAnyIgnoreCase(queue.getRunMode(), ApiRunMode.SCENARIO.name(), ApiRunMode.SCENARIO_PLAN.name(), ApiRunMode.SCHEDULE_SCENARIO_PLAN.name(), ApiRunMode.SCHEDULE_SCENARIO.name(), ApiRunMode.JENKINS_SCENARIO_PLAN.name())) { if (StringUtils.equalsAnyIgnoreCase(queue.getRunMode(), ApiRunMode.SCENARIO.name(), ApiRunMode.SCENARIO_PLAN.name(), ApiRunMode.SCHEDULE_SCENARIO_PLAN.name(), ApiRunMode.SCHEDULE_SCENARIO.name(), ApiRunMode.JENKINS_SCENARIO_PLAN.name())) {
redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(item.getReportId()));
ApiScenarioReportWithBLOBs report = apiScenarioReportMapper.selectByPrimaryKey(item.getReportId()); ApiScenarioReportWithBLOBs report = apiScenarioReportMapper.selectByPrimaryKey(item.getReportId());
// 报告已经被删除则队列也删除 // 报告已经被删除则队列也删除
if (report == null) { if (report == null) {
@ -386,7 +388,6 @@ public class ApiExecutionQueueService {
if (report != null && StringUtils.equalsAnyIgnoreCase(report.getStatus(), TestPlanReportStatus.RUNNING.name()) && report.getUpdateTime() < timeout) { if (report != null && StringUtils.equalsAnyIgnoreCase(report.getStatus(), TestPlanReportStatus.RUNNING.name()) && report.getUpdateTime() < timeout) {
report.setStatus(ApiReportStatus.ERROR.name()); report.setStatus(ApiReportStatus.ERROR.name());
apiScenarioReportMapper.updateByPrimaryKeySelective(report); apiScenarioReportMapper.updateByPrimaryKeySelective(report);
LoggerUtil.info("超时处理报告：" + report.getId()); LoggerUtil.info("超时处理报告：" + report.getId());
if (queue != null && StringUtils.equalsIgnoreCase(item.getType(), RunModeConstants.SERIAL.toString())) { if (queue != null && StringUtils.equalsIgnoreCase(item.getType(), RunModeConstants.SERIAL.toString())) {
// 删除串行资源锁 // 删除串行资源锁
@ -405,6 +406,7 @@ public class ApiExecutionQueueService {
} }
} }
} else { } else {
redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(item.getReportId()));
// 用例/接口超时结果处理 // 用例/接口超时结果处理
ApiDefinitionExecResultWithBLOBs result = apiDefinitionExecResultMapper.selectByPrimaryKey(item.getReportId()); ApiDefinitionExecResultWithBLOBs result = apiDefinitionExecResultMapper.selectByPrimaryKey(item.getReportId());
if (result != null && StringUtils.equalsAnyIgnoreCase(result.getStatus(), TestPlanReportStatus.RUNNING.name())) { if (result != null && StringUtils.equalsAnyIgnoreCase(result.getStatus(), TestPlanReportStatus.RUNNING.name())) {

30
api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java
View File

@ -26,7 +26,6 @@ import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.jorphan.collections.HashTree; import org.apache.jorphan.collections.HashTree;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
@ -54,7 +53,7 @@ public class ApiJMeterFileService {
@Resource @Resource
private PluginMapper pluginMapper; private PluginMapper pluginMapper;
@Resource @Resource
private RedisTemplate<String, Object> redisTemplate; private RedisTemplateService redisTemplateService;
// 接口测试 用例/接口 // 接口测试 用例/接口
private static final List<String> CASE_MODES = new ArrayList<>() {{ private static final List<String> CASE_MODES = new ArrayList<>() {{
@ -249,10 +248,9 @@ public class ApiJMeterFileService {
Map<String, byte[]> multipartFiles = this.getMultipartFiles(testId, hashTree); Map<String, byte[]> multipartFiles = this.getMultipartFiles(testId, hashTree);
转为解析jmx中附件节点，赋予相关信息(例如文件关联类型、路径、更新时间等),并将文件信息存储在redis中，为了进行连接ms下载时的安全校验 转为解析jmx中附件节点，赋予相关信息(例如文件关联类型、路径、更新时间等),并将文件信息存储在redis中，为了进行连接ms下载时的安全校验
*/ */
List<AttachmentBodyFile> attachmentBodyFileList = new ArrayList<>(); List<AttachmentBodyFile> attachmentBodyFileList = ApiFileUtil.getExecuteFileForNode(hashTree, reportId);
ApiFileUtil.formatFilePathForNode(hashTree, testId, attachmentBodyFileList);
if (CollectionUtils.isNotEmpty(attachmentBodyFileList)) { if (CollectionUtils.isNotEmpty(attachmentBodyFileList)) {
redisTemplate.opsForValue().set(JmxFileUtil.REDIS_JMX_FILE_PREFIX + reportId, JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList)); redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteFileKeyInRedis(reportId), JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList));
} }
String jmx = new MsTestPlan().getJmx(hashTree); String jmx = new MsTestPlan().getJmx(hashTree);
@ -285,21 +283,21 @@ public class ApiJMeterFileService {
} }
} }
public byte[] zipFilesToByteArray(BodyFileRequest request) { /**
* 打包ms本地文件
*
* @param request
* @return
*/
public byte[] zipLocalFilesToByteArray(BodyFileRequest request) {
Map<String, byte[]> files = new LinkedHashMap<>(); Map<String, byte[]> files = new LinkedHashMap<>();
if (CollectionUtils.isNotEmpty(request.getBodyFiles())) { if (CollectionUtils.isNotEmpty(request.getBodyFiles())) {
//获取要下载的合法文件
List<BodyFile> bodyFiles = this.getLegalFiles(request); List<BodyFile> bodyFiles = this.getLegalFiles(request);
LoggerUtil.info("开始从三方仓库下载文件");
HashTreeUtil.downFile(bodyFiles, files, fileMetadataService); HashTreeUtil.downFile(bodyFiles, files, fileMetadataService);
LoggerUtil.info("从三方仓库下载文件");
for (BodyFile bodyFile : bodyFiles) { for (BodyFile bodyFile : bodyFiles) {
File file = new File(bodyFile.getName()); File file = new File(bodyFile.getName());
if (!file.exists()) { if (file.exists() && StringUtils.startsWith(file.getPath(), FileUtils.ROOT_DIR)) {
// 从MinIO下载
ApiFileUtil.downloadFile(bodyFile.getId(), bodyFile.getName());
file = new File(bodyFile.getName());
}
if (file != null && file.exists() && StringUtils.startsWith(file.getPath(), FileUtils.ROOT_DIR)) {
byte[] fileByte = FileUtils.fileToByte(file); byte[] fileByte = FileUtils.fileToByte(file);
if (fileByte != null) { if (fileByte != null) {
files.put(file.getAbsolutePath(), fileByte); files.put(file.getAbsolutePath(), fileByte);
@ -322,8 +320,10 @@ public class ApiJMeterFileService {
private List<BodyFile> getLegalFiles(BodyFileRequest request) { private List<BodyFile> getLegalFiles(BodyFileRequest request) {
List<BodyFile> returnList = new ArrayList<>(); List<BodyFile> returnList = new ArrayList<>();
Object jmxFileInfoObj = redisTemplate.opsForValue().get(JmxFileUtil.REDIS_JMX_FILE_PREFIX + request.getReportId()); Object jmxFileInfoObj = redisTemplateService.get(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId()));
List<AttachmentBodyFile> fileInJmx = JmxFileUtil.formatRedisJmxFileString(jmxFileInfoObj); List<AttachmentBodyFile> fileInJmx = JmxFileUtil.formatRedisJmxFileString(jmxFileInfoObj);
redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId()));
if (CollectionUtils.isNotEmpty(request.getBodyFiles())) { if (CollectionUtils.isNotEmpty(request.getBodyFiles())) {
request.getBodyFiles().forEach(attachmentBodyFile -> { request.getBodyFiles().forEach(attachmentBodyFile -> {
for (AttachmentBodyFile jmxFile : fileInJmx) { for (AttachmentBodyFile jmxFile : fileInJmx) {

11
api-test/backend/src/main/java/io/metersphere/service/RedisTemplateService.java
View File

@ -1,10 +1,10 @@
package io.metersphere.service; package io.metersphere.service;
import io.metersphere.commons.utils.LogUtil; import io.metersphere.commons.utils.LogUtil;
import jakarta.annotation.Resource;
import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import jakarta.annotation.Resource;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@Service @Service
@ -31,6 +31,15 @@ public class RedisTemplateService {
} }
} }
public Object get(String key) {
try {
return redisTemplate.opsForValue().get(key);
} catch (Exception e) {
LogUtil.error(e);
}
return null;
}
public boolean delete(String key) { public boolean delete(String key) {
try { try {
return redisTemplate.delete(key); return redisTemplate.delete(key);