From d0b7823d1b8980c7c7d0aa0983927cd430554060 Mon Sep 17 00:00:00 2001 From: song-tianyang Date: Thu, 9 Feb 2023 14:58:59 +0800 Subject: [PATCH] =?UTF-8?q?refactor(=E6=8E=A5=E5=8F=A3=E6=B5=8B=E8=AF=95):?= =?UTF-8?q?=20=E7=A6=81=E7=94=A8=E6=9C=AC=E5=9C=B0=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E6=97=B6=E6=8E=A5=E5=8F=A3=E6=89=A7=E8=A1=8C=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E9=99=84=E4=BB=B6=E4=BC=A0=E8=BE=93=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --story=1011099 --user=宋天阳 执行附件本地缓存，避免每次执行都下载文件性能开销比较大 https://www.tapd.cn/55049933/s/1332544 --- .../metersphere/api/jmeter/JMeterService.java | 44 ++++++----- .../api/jmeter/KafkaListenerTask.java | 8 +- .../api/jmeter/MsApiBackendListener.java | 14 ++-- .../api/jmeter/MsKafkaListener.java | 6 +- .../api/jmeter/utils/JmxFileUtil.java | 10 ++- .../commons/utils/ApiFileUtil.java | 79 ++----------------- .../controller/ApiJMeterFileController.java | 8 +- .../service/ApiExecutionQueueService.java | 6 +- .../service/ApiJMeterFileService.java | 30 +++---- .../service/RedisTemplateService.java | 11 ++- 10 files changed, 87 insertions(+), 129 deletions(-) diff --git a/api-test/backend/src/main/java/io/metersphere/api/jmeter/JMeterService.java b/api-test/backend/src/main/java/io/metersphere/api/jmeter/JMeterService.java index 9d897b9dfc..09be91a42d 100644 --- a/api-test/backend/src/main/java/io/metersphere/api/jmeter/JMeterService.java +++ b/api-test/backend/src/main/java/io/metersphere/api/jmeter/JMeterService.java @@ -5,6 +5,7 @@ import io.metersphere.api.dto.definition.request.ElementUtil; import io.metersphere.api.dto.definition.request.MsTestPlan; import io.metersphere.api.exec.engine.EngineFactory; import io.metersphere.api.exec.queue.ExecThreadPoolExecutor; +import io.metersphere.api.jmeter.utils.JmxFileUtil; import io.metersphere.api.jmeter.utils.ServerConfig; import io.metersphere.api.jmeter.utils.SmoothWeighted; import io.metersphere.base.domain.TestResource; @@ -12,22 +13,17 @@ import io.metersphere.commons.config.KafkaConfig; import io.metersphere.commons.constants.ApiRunMode; import io.metersphere.commons.constants.ExtendedParameter; import io.metersphere.commons.exception.MSException; -import io.metersphere.commons.utils.FixedCapacityUtil; -import io.metersphere.commons.utils.GenerateHashTreeUtil; -import io.metersphere.commons.utils.HashTreeUtil; -import io.metersphere.commons.utils.JSON; +import io.metersphere.commons.utils.*; import io.metersphere.config.JmeterProperties; import io.metersphere.constants.BackendListenerConstants; import io.metersphere.constants.RunModeConstants; -import io.metersphere.dto.JmeterRunRequestDTO; -import io.metersphere.dto.NodeDTO; -import io.metersphere.dto.PluginConfigDTO; -import io.metersphere.dto.RunModeConfigDTO; +import io.metersphere.dto.*; import io.metersphere.engine.Engine; import io.metersphere.jmeter.JMeterBase; import io.metersphere.jmeter.LocalRunner; import io.metersphere.service.ApiPoolDebugService; import io.metersphere.service.PluginService; +import io.metersphere.service.RedisTemplateService; import io.metersphere.service.RemakeReportService; import io.metersphere.utils.LoggerUtil; import jakarta.annotation.PostConstruct; @@ -53,8 +49,6 @@ import java.util.List; @Service public class JMeterService { public static final String BASE_URL = "http://%s:%d"; - public static final String POOL = "POOL"; - @Resource private JmeterProperties jmeterProperties; @Resource @@ -62,6 +56,8 @@ public class JMeterService { @Resource private RedisTemplate redisTemplate; @Resource + private RedisTemplateService redisTemplateService; + @Resource private RemakeReportService remakeReportService; @Resource private ExecThreadPoolExecutor execThreadPoolExecutor; @@ -165,16 +161,21 @@ public class JMeterService { // 缓存调试脚本 if (request.getHashTree() != null) { ElementUtil.coverArguments(request.getHashTree()); - String key = StringUtils.join(request.getReportId(), "-", request.getTestId()); - redisTemplate.opsForValue().set(key, new MsTestPlan().getJmx(request.getHashTree())); + //解析HashTree里的文件信息 + List attachmentBodyFileList = ApiFileUtil.getExecuteFileForNode(request.getHashTree(), request.getReportId()); + if (CollectionUtils.isNotEmpty(attachmentBodyFileList)) { + redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId()), JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList)); + } + + redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteScriptKey(request.getReportId(), request.getTestId()), new MsTestPlan().getJmx(request.getHashTree())); } LoggerUtil.info("开始发送请求[ " + request.getTestId() + " ] 到K8S节点执行", request.getReportId()); final Engine engine = EngineFactory.createApiEngine(request); engine.start(); } catch (Exception e) { remakeReportService.testEnded(request, e.getMessage()); - String key = StringUtils.join(request.getReportId(), "-", request.getTestId()); - redisTemplate.delete(key); + redisTemplateService.delete(JmxFileUtil.getExecuteScriptKey(request.getReportId(), request.getTestId())); + redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId())); LoggerUtil.error("调用K8S执行请求[ " + request.getTestId() + " ]失败：", request.getReportId(), e); } } else if ((MapUtils.isNotEmpty(request.getExtendedParameters()) @@ -193,16 +194,23 @@ public class JMeterService { if (request.getHashTree() != null) { // 过程变量处理 ElementUtil.coverArguments(request.getHashTree()); - String key = StringUtils.join(request.getReportId(), "-", request.getTestId()); - redisTemplate.opsForValue().set(key, new MsTestPlan().getJmx(request.getHashTree())); + + + //解析HashTree里的文件信息 + List attachmentBodyFileList = ApiFileUtil.getExecuteFileForNode(request.getHashTree(), request.getReportId()); + if (CollectionUtils.isNotEmpty(attachmentBodyFileList)) { + redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId()), JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList)); + } + + redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteScriptKey(request.getReportId(), request.getTestId()), new MsTestPlan().getJmx(request.getHashTree())); request.setHashTree(null); } apiPoolDebugService.run(request, resources); } catch (Exception e) { LoggerUtil.error(e); remakeReportService.remake(request); - String key = StringUtils.join(request.getReportId(), "-", request.getTestId()); - redisTemplate.delete(key); + redisTemplateService.delete(JmxFileUtil.getExecuteScriptKey(request.getReportId(), request.getTestId())); + redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId())); LoggerUtil.error("发送请求[ " + request.getTestId() + " ] 执行失败,进行数据回滚：", request.getReportId(), e); MSException.throwException("调用资源池执行失败，请检查资源池是否配置正常"); } diff --git a/api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java b/api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java index 69aa2eeafc..188e322011 100644 --- a/api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java +++ b/api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java @@ -8,6 +8,7 @@ import io.metersphere.commons.constants.ExtendedParameter; import io.metersphere.commons.utils.JSON; import io.metersphere.dto.ResultDTO; import io.metersphere.service.ApiExecutionQueueService; +import io.metersphere.service.RedisTemplateService; import io.metersphere.service.TestResultService; import io.metersphere.utils.LoggerUtil; import lombok.Data; @@ -15,7 +16,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.springframework.data.redis.core.RedisTemplate; import java.util.*; @@ -25,7 +25,7 @@ public class KafkaListenerTask implements Runnable { private ApiExecutionQueueService apiExecutionQueueService; private TestResultService testResultService; - private RedisTemplate redisTemplate; + private RedisTemplateService redisTemplateService; private static final Map RUN_MODE_MAP = new HashMap() {{ this.put(ApiRunMode.SCHEDULE_API_PLAN.name(), "schedule-task"); @@ -58,9 +58,7 @@ public class KafkaListenerTask implements Runnable { if (dto == null) { return; } - if (redisTemplate != null) { - redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId()); - } + redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(dto.getReportId())); if (dto.getArbitraryData() != null && dto.getArbitraryData().containsKey(ExtendedParameter.TEST_END) && (Boolean) dto.getArbitraryData().get(ExtendedParameter.TEST_END)) { diff --git a/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java b/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java index 9a9129bc9c..f07fa2c7e9 100644 --- a/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java +++ b/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java @@ -12,6 +12,7 @@ import io.metersphere.constants.RunModeConstants; import io.metersphere.dto.ResultDTO; import io.metersphere.jmeter.JMeterBase; import io.metersphere.service.ApiExecutionQueueService; +import io.metersphere.service.RedisTemplateService; import io.metersphere.service.TestResultService; import io.metersphere.utils.LoggerUtil; import io.metersphere.utils.RetryResultUtil; @@ -21,7 +22,6 @@ import org.apache.jmeter.samplers.SampleResult; import org.apache.jmeter.services.FileServer; import org.apache.jmeter.visualizers.backend.AbstractBackendListenerClient; import org.apache.jmeter.visualizers.backend.BackendListenerContext; -import org.springframework.data.redis.core.RedisTemplate; import java.io.Serializable; import java.util.LinkedHashMap; @@ -38,7 +38,7 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen private ResultVO resultVO; @Resource - private RedisTemplate redisTemplate; + private RedisTemplateService redisTemplateService; /** * 参数初始化方法 @@ -64,9 +64,7 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen if (dto.isRetryEnable()) { queues.addAll(sampleResults); } else { - if (redisTemplate != null) { - redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId()); - } + redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(dto.getReportId())); if (!StringUtils.equals(dto.getReportType(), RunModeConstants.SET_REPORT.toString())) { dto.setConsole(FixedCapacityUtil.getJmeterLogger(getReportId(), false)); @@ -85,10 +83,8 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen try { LoggerUtil.info("进入TEST-END处理报告" + dto.getRunMode(), dto.getReportId()); - if (redisTemplate != null) { - redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId()); - } - + redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(dto.getReportId())); + super.teardownTest(context); // 获取执行日志 if (!StringUtils.equals(dto.getReportType(), RunModeConstants.SET_REPORT.toString())) { diff --git a/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java b/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java index 487605a020..a9810c5a61 100644 --- a/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java +++ b/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java @@ -5,6 +5,7 @@ import io.metersphere.commons.constants.ApiRunMode; import io.metersphere.commons.constants.KafkaTopicConstants; import io.metersphere.commons.utils.*; import io.metersphere.service.ApiExecutionQueueService; +import io.metersphere.service.RedisTemplateService; import io.metersphere.service.TestResultService; import io.metersphere.service.definition.ApiDefinitionEnvService; import io.metersphere.utils.LoggerUtil; @@ -13,7 +14,6 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.springframework.context.annotation.Configuration; -import org.springframework.data.redis.core.RedisTemplate; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.support.Acknowledgment; @@ -40,7 +40,7 @@ public class MsKafkaListener { private final static int WORK_QUEUE_SIZE = 10000; @Resource - private RedisTemplate redisTemplate; + private RedisTemplateService redisTemplateService; private final ThreadPoolExecutor threadPool = new ThreadPoolExecutor( CORE_POOL_SIZE, @@ -59,7 +59,7 @@ public class MsKafkaListener { task.setApiExecutionQueueService(apiExecutionQueueService); task.setTestResultService(testResultService); task.setRecord(item); - task.setRedisTemplate(redisTemplate); + task.setRedisTemplateService(redisTemplateService); threadPool.execute(task); }); JvmUtil.memoryInfo(); diff --git a/api-test/backend/src/main/java/io/metersphere/api/jmeter/utils/JmxFileUtil.java b/api-test/backend/src/main/java/io/metersphere/api/jmeter/utils/JmxFileUtil.java index 3205d12f52..76b8e65d2d 100644 --- a/api-test/backend/src/main/java/io/metersphere/api/jmeter/utils/JmxFileUtil.java +++ b/api-test/backend/src/main/java/io/metersphere/api/jmeter/utils/JmxFileUtil.java @@ -12,7 +12,7 @@ import java.util.List; import java.util.Map; public class JmxFileUtil { - public static final String REDIS_JMX_FILE_PREFIX = "JMX.FILE."; + public static final String REDIS_JMX_EXECUTE_FILE_PREFIX = "JMX.FILE."; private static final String JMX_INFO_KEY_ID = "id"; private static final String JMX_INFO_KEY_FILE_PATH = "filePath"; @@ -21,6 +21,14 @@ public class JmxFileUtil { private static final String JMX_INFO_KEY_FILE_STORAGE = "storage"; + public static String getExecuteFileKeyInRedis(String reportId) { + return StringUtils.join(REDIS_JMX_EXECUTE_FILE_PREFIX, reportId); + } + + public static String getExecuteScriptKey(String reportId, String testId) { + return StringUtils.join(reportId, "-", testId); + } + public static String getRedisJmxFileString(List listFile) { List> jmxFileList = new ArrayList<>(); listFile.forEach(file -> { diff --git a/api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java b/api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java index 8f7aff8ffd..445a6d34e0 100644 --- a/api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java +++ b/api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java @@ -23,6 +23,7 @@ import org.apache.jorphan.collections.HashTree; import org.springframework.web.multipart.MultipartFile; import java.io.File; +import java.util.ArrayList; import java.util.List; public class ApiFileUtil extends FileUtils { @@ -110,7 +111,13 @@ public class ApiFileUtil extends FileUtils { } } - public static void formatFilePathForNode(HashTree tree, String reportId, List fileList) { + public static List getExecuteFileForNode(HashTree tree, String reportId) { + List fileList = new ArrayList<>(); + formatFilePathForNode(tree, reportId, fileList); + return fileList; + } + + private static void formatFilePathForNode(HashTree tree, String reportId, List fileList) { if (tree != null) { if (fileMetadataService == null) { fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class); @@ -217,76 +224,6 @@ public class ApiFileUtil extends FileUtils { return StringUtils.join(BODY_FILE_DIR, File.separator, reportId, File.separator, fileName); } - // public static void formatFilePathForNode(HashTree tree, String reportId, List fileList) { - // if (fileMetadataService == null) { - // fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class); - // } - // for (Object key : tree.keySet()) { - // if (key == null) { - // continue; - // } - // HashTree node = tree.get(key); - // if (key instanceof HTTPSamplerProxy) { - // formatHttpFilePathForNode(key, reportId, fileList); - // } else if (key instanceof CSVDataSet) { - // formatCsvFilePathForNode(key, reportId, fileList); - // } - // if (node != null) { - // formatFilePathForNode(node, reportId, fileList); - // } - // } - // } - - // private static void formatCsvFilePathForNode(Object key, String reportId) { - // CSVDataSet source = (CSVDataSet) key; - // if (StringUtils.isNotEmpty(source.getPropertyAsString(ElementConstants.FILENAME))) { - // if (source.getPropertyAsBoolean(ElementConstants.IS_REF)) { - // FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById( - // source.getPropertyAsString(ElementConstants.FILE_ID)); - // if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) { - // String path = getFilePathInJxm(reportId, fileMetadata.getName()); - // ((CSVDataSet) key).setProperty(ElementConstants.FILENAME, path); - // ((CSVDataSet) key).setProperty(ElementConstants.FILE_STORAGE, fileMetadata.getStorage()); - // ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_NAME, fileMetadata.getName()); - // ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_UPDATE_TIME, fileMetadata.getUpdateTime()); - // ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_PROJECT_ID, fileMetadata.getProjectId()); - // if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) { - // ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_ATTACH_INFO, fileMetadata.getAttachInfo()); - // } - // } - // } - // } - // } - - // private static void formatHttpFilePathForNode(Object key, String reportId, List fileList) { - // if (key == null) { - // return; - // } - // HTTPSamplerProxy source = (HTTPSamplerProxy) key; - // for (HTTPFileArg httpFileArg : source.getHTTPFiles()) { - // if (httpFileArg.getPropertyAsBoolean(ElementConstants.IS_REF)) { - // FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById( - // httpFileArg.getPropertyAsString(ElementConstants.FILE_ID)); - // if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) { - // String path = getFilePathInJxm(reportId, fileMetadata.getName()); - // httpFileArg.setPath(path); - // httpFileArg.setProperty(ElementConstants.FILE_STORAGE, fileMetadata.getStorage()); - // httpFileArg.setProperty(ElementConstants.REF_FILE_NAME, fileMetadata.getName()); - // httpFileArg.setProperty(ElementConstants.REF_FILE_UPDATE_TIME, fileMetadata.getUpdateTime()); - // httpFileArg.setProperty(ElementConstants.REF_FILE_PROJECT_ID, fileMetadata.getProjectId()); - // if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) { - // httpFileArg.setProperty(ElementConstants.REF_FILE_ATTACH_INFO, fileMetadata.getAttachInfo()); - // } - // - // fileList.add(new AttachmentBodyFile() {{ - // this.setFileMetadataId(fileMetadata.getId()); - // }}); - // } - // } - // } - // - // - /** * 获取当前jmx 涉及到的文件 执行时 * diff --git a/api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java b/api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java index 5a65d064c4..c9a5e4debf 100644 --- a/api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java +++ b/api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java @@ -2,10 +2,10 @@ package io.metersphere.controller; import io.metersphere.api.dto.BodyFileRequest; import io.metersphere.api.jmeter.JMeterThreadUtils; +import io.metersphere.api.jmeter.utils.JmxFileUtil; import io.metersphere.service.ApiJMeterFileService; import io.metersphere.utils.LoggerUtil; import jakarta.annotation.Resource; -import org.apache.commons.lang3.StringUtils; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; @@ -68,8 +68,8 @@ public class ApiJMeterFileController { @PostMapping("download/files") - public ResponseEntity downloadJmeterFiles(@RequestBody BodyFileRequest request) { - byte[] bytes = apiJmeterFileService.zipFilesToByteArray(request); + public ResponseEntity downloadJmeterLocalFiles(@RequestBody BodyFileRequest request) { + byte[] bytes = apiJmeterFileService.zipLocalFilesToByteArray(request); return ResponseEntity.ok() .contentType(MediaType.parseMediaType("application/octet-stream")) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + request.getReportId() + ".zip\"") @@ -78,7 +78,7 @@ public class ApiJMeterFileController { @GetMapping("get-script") public String getScript(@RequestParam("reportId") String reportId, @RequestParam("testId") String testId) { - String key = StringUtils.join(reportId, "-", testId); + String key = JmxFileUtil.getExecuteScriptKey(reportId, testId); LoggerUtil.info("获取执行脚本", key); Object script = redisTemplate.opsForValue().get(key); redisTemplate.delete(key); diff --git a/api-test/backend/src/main/java/io/metersphere/service/ApiExecutionQueueService.java b/api-test/backend/src/main/java/io/metersphere/service/ApiExecutionQueueService.java index baf890e615..2416f0f720 100644 --- a/api-test/backend/src/main/java/io/metersphere/service/ApiExecutionQueueService.java +++ b/api-test/backend/src/main/java/io/metersphere/service/ApiExecutionQueueService.java @@ -6,6 +6,7 @@ import io.metersphere.api.exec.queue.DBTestQueue; import io.metersphere.api.exec.scenario.ApiScenarioSerialService; import io.metersphere.api.jmeter.JMeterService; import io.metersphere.api.jmeter.JMeterThreadUtils; +import io.metersphere.api.jmeter.utils.JmxFileUtil; import io.metersphere.base.domain.*; import io.metersphere.base.mapper.*; import io.metersphere.base.mapper.ext.BaseApiExecutionQueueMapper; @@ -23,6 +24,7 @@ import io.metersphere.dto.ResultDTO; import io.metersphere.dto.RunModeConfigDTO; import io.metersphere.service.scenario.ApiScenarioReportService; import io.metersphere.utils.LoggerUtil; +import jakarta.annotation.Resource; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.BooleanUtils; @@ -32,7 +34,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import jakarta.annotation.Resource; import java.util.*; import java.util.stream.Collectors; @@ -377,6 +378,7 @@ public class ApiExecutionQueueService { dto.setQueueId(item.getQueueId()); dto.setTestId(item.getTestId()); if (StringUtils.equalsAnyIgnoreCase(queue.getRunMode(), ApiRunMode.SCENARIO.name(), ApiRunMode.SCENARIO_PLAN.name(), ApiRunMode.SCHEDULE_SCENARIO_PLAN.name(), ApiRunMode.SCHEDULE_SCENARIO.name(), ApiRunMode.JENKINS_SCENARIO_PLAN.name())) { + redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(item.getReportId())); ApiScenarioReportWithBLOBs report = apiScenarioReportMapper.selectByPrimaryKey(item.getReportId()); // 报告已经被删除则队列也删除 if (report == null) { @@ -386,7 +388,6 @@ public class ApiExecutionQueueService { if (report != null && StringUtils.equalsAnyIgnoreCase(report.getStatus(), TestPlanReportStatus.RUNNING.name()) && report.getUpdateTime() < timeout) { report.setStatus(ApiReportStatus.ERROR.name()); apiScenarioReportMapper.updateByPrimaryKeySelective(report); - LoggerUtil.info("超时处理报告：" + report.getId()); if (queue != null && StringUtils.equalsIgnoreCase(item.getType(), RunModeConstants.SERIAL.toString())) { // 删除串行资源锁 @@ -405,6 +406,7 @@ public class ApiExecutionQueueService { } } } else { + redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(item.getReportId())); // 用例/接口超时结果处理 ApiDefinitionExecResultWithBLOBs result = apiDefinitionExecResultMapper.selectByPrimaryKey(item.getReportId()); if (result != null && StringUtils.equalsAnyIgnoreCase(result.getStatus(), TestPlanReportStatus.RUNNING.name())) { diff --git a/api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java b/api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java index 06e6e6cc85..95705870bd 100644 --- a/api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java +++ b/api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java @@ -26,7 +26,6 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.jorphan.collections.HashTree; -import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; import java.io.ByteArrayOutputStream; @@ -54,7 +53,7 @@ public class ApiJMeterFileService { @Resource private PluginMapper pluginMapper; @Resource - private RedisTemplate redisTemplate; + private RedisTemplateService redisTemplateService; // 接口测试 用例/接口 private static final List CASE_MODES = new ArrayList<>() {{ @@ -249,10 +248,9 @@ public class ApiJMeterFileService { Map multipartFiles = this.getMultipartFiles(testId, hashTree); 转为解析jmx中附件节点，赋予相关信息(例如文件关联类型、路径、更新时间等),并将文件信息存储在redis中，为了进行连接ms下载时的安全校验 */ - List attachmentBodyFileList = new ArrayList<>(); - ApiFileUtil.formatFilePathForNode(hashTree, testId, attachmentBodyFileList); + List attachmentBodyFileList = ApiFileUtil.getExecuteFileForNode(hashTree, reportId); if (CollectionUtils.isNotEmpty(attachmentBodyFileList)) { - redisTemplate.opsForValue().set(JmxFileUtil.REDIS_JMX_FILE_PREFIX + reportId, JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList)); + redisTemplateService.setIfAbsent(JmxFileUtil.getExecuteFileKeyInRedis(reportId), JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList)); } String jmx = new MsTestPlan().getJmx(hashTree); @@ -285,21 +283,21 @@ public class ApiJMeterFileService { } } - public byte[] zipFilesToByteArray(BodyFileRequest request) { + /** + * 打包ms本地文件 + * + * @param request + * @return + */ + public byte[] zipLocalFilesToByteArray(BodyFileRequest request) { Map files = new LinkedHashMap<>(); if (CollectionUtils.isNotEmpty(request.getBodyFiles())) { + //获取要下载的合法文件 List bodyFiles = this.getLegalFiles(request); - LoggerUtil.info("开始从三方仓库下载文件"); HashTreeUtil.downFile(bodyFiles, files, fileMetadataService); - LoggerUtil.info("从三方仓库下载文件"); for (BodyFile bodyFile : bodyFiles) { File file = new File(bodyFile.getName()); - if (!file.exists()) { - // 从MinIO下载 - ApiFileUtil.downloadFile(bodyFile.getId(), bodyFile.getName()); - file = new File(bodyFile.getName()); - } - if (file != null && file.exists() && StringUtils.startsWith(file.getPath(), FileUtils.ROOT_DIR)) { + if (file.exists() && StringUtils.startsWith(file.getPath(), FileUtils.ROOT_DIR)) { byte[] fileByte = FileUtils.fileToByte(file); if (fileByte != null) { files.put(file.getAbsolutePath(), fileByte); @@ -322,8 +320,10 @@ public class ApiJMeterFileService { private List getLegalFiles(BodyFileRequest request) { List returnList = new ArrayList<>(); - Object jmxFileInfoObj = redisTemplate.opsForValue().get(JmxFileUtil.REDIS_JMX_FILE_PREFIX + request.getReportId()); + Object jmxFileInfoObj = redisTemplateService.get(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId())); List fileInJmx = JmxFileUtil.formatRedisJmxFileString(jmxFileInfoObj); + redisTemplateService.delete(JmxFileUtil.getExecuteFileKeyInRedis(request.getReportId())); + if (CollectionUtils.isNotEmpty(request.getBodyFiles())) { request.getBodyFiles().forEach(attachmentBodyFile -> { for (AttachmentBodyFile jmxFile : fileInJmx) { diff --git a/api-test/backend/src/main/java/io/metersphere/service/RedisTemplateService.java b/api-test/backend/src/main/java/io/metersphere/service/RedisTemplateService.java index a88c0a573f..2be3f4817a 100644 --- a/api-test/backend/src/main/java/io/metersphere/service/RedisTemplateService.java +++ b/api-test/backend/src/main/java/io/metersphere/service/RedisTemplateService.java @@ -1,10 +1,10 @@ package io.metersphere.service; import io.metersphere.commons.utils.LogUtil; +import jakarta.annotation.Resource; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; -import jakarta.annotation.Resource; import java.util.concurrent.TimeUnit; @Service @@ -31,6 +31,15 @@ public class RedisTemplateService { } } + public Object get(String key) { + try { + return redisTemplate.opsForValue().get(key); + } catch (Exception e) { + LogUtil.error(e); + } + return null; + } + public boolean delete(String key) { try { return redisTemplate.delete(key);