diff --git a/api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java b/api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java index 2744034f26..69aa2eeafc 100644 --- a/api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java +++ b/api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java @@ -2,6 +2,7 @@ package io.metersphere.api.jmeter; import com.fasterxml.jackson.core.type.TypeReference; import io.metersphere.api.exec.queue.PoolExecBlockingQueueUtil; +import io.metersphere.api.jmeter.utils.JmxFileUtil; import io.metersphere.commons.constants.ApiRunMode; import io.metersphere.commons.constants.ExtendedParameter; import io.metersphere.commons.utils.JSON; @@ -14,6 +15,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.springframework.data.redis.core.RedisTemplate; import java.util.*; @@ -22,6 +24,9 @@ public class KafkaListenerTask implements Runnable { private ConsumerRecord record; private ApiExecutionQueueService apiExecutionQueueService; private TestResultService testResultService; + + private RedisTemplate redisTemplate; + private static final Map RUN_MODE_MAP = new HashMap() {{ this.put(ApiRunMode.SCHEDULE_API_PLAN.name(), "schedule-task"); this.put(ApiRunMode.JENKINS_API_PLAN.name(), "schedule-task"); @@ -53,6 +58,10 @@ public class KafkaListenerTask implements Runnable { if (dto == null) { return; } + if (redisTemplate != null) { + redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId()); + } + if (dto.getArbitraryData() != null && dto.getArbitraryData().containsKey(ExtendedParameter.TEST_END) && (Boolean) dto.getArbitraryData().get(ExtendedParameter.TEST_END)) { resultDTOS.add(dto); diff --git a/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java b/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java index 55579c416b..9a9129bc9c 100644 --- a/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java +++ b/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java @@ -2,6 +2,7 @@ package io.metersphere.api.jmeter; import io.metersphere.api.exec.queue.PoolExecBlockingQueueUtil; +import io.metersphere.api.jmeter.utils.JmxFileUtil; import io.metersphere.api.jmeter.utils.ReportStatusUtil; import io.metersphere.commons.constants.CommonConstants; import io.metersphere.commons.utils.*; @@ -14,11 +15,13 @@ import io.metersphere.service.ApiExecutionQueueService; import io.metersphere.service.TestResultService; import io.metersphere.utils.LoggerUtil; import io.metersphere.utils.RetryResultUtil; +import jakarta.annotation.Resource; import org.apache.commons.lang3.StringUtils; import org.apache.jmeter.samplers.SampleResult; import org.apache.jmeter.services.FileServer; import org.apache.jmeter.visualizers.backend.AbstractBackendListenerClient; import org.apache.jmeter.visualizers.backend.BackendListenerContext; +import org.springframework.data.redis.core.RedisTemplate; import java.io.Serializable; import java.util.LinkedHashMap; @@ -34,6 +37,9 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen // 当前场景报告/用例结果状态 private ResultVO resultVO; + @Resource + private RedisTemplate redisTemplate; + /** * 参数初始化方法 */ @@ -58,6 +64,10 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen if (dto.isRetryEnable()) { queues.addAll(sampleResults); } else { + if (redisTemplate != null) { + redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId()); + } + if (!StringUtils.equals(dto.getReportType(), RunModeConstants.SET_REPORT.toString())) { dto.setConsole(FixedCapacityUtil.getJmeterLogger(getReportId(), false)); } @@ -74,6 +84,11 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen public void teardownTest(BackendListenerContext context) { try { LoggerUtil.info("进入TEST-END处理报告" + dto.getRunMode(), dto.getReportId()); + + if (redisTemplate != null) { + redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId()); + } + super.teardownTest(context); // 获取执行日志 if (!StringUtils.equals(dto.getReportType(), RunModeConstants.SET_REPORT.toString())) { diff --git a/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java b/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java index c6a68cf7b7..487605a020 100644 --- a/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java +++ b/api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java @@ -8,14 +8,15 @@ import io.metersphere.service.ApiExecutionQueueService; import io.metersphere.service.TestResultService; import io.metersphere.service.definition.ApiDefinitionEnvService; import io.metersphere.utils.LoggerUtil; +import jakarta.annotation.Resource; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.core.RedisTemplate; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.support.Acknowledgment; -import jakarta.annotation.Resource; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -38,6 +39,9 @@ public class MsKafkaListener { // 线程池所使用的缓冲队列大小 private final static int WORK_QUEUE_SIZE = 10000; + @Resource + private RedisTemplate redisTemplate; + private final ThreadPoolExecutor threadPool = new ThreadPoolExecutor( CORE_POOL_SIZE, MAX_POOL_SIZE, @@ -55,6 +59,7 @@ public class MsKafkaListener { task.setApiExecutionQueueService(apiExecutionQueueService); task.setTestResultService(testResultService); task.setRecord(item); + task.setRedisTemplate(redisTemplate); threadPool.execute(task); }); JvmUtil.memoryInfo(); diff --git a/api-test/backend/src/main/java/io/metersphere/api/jmeter/utils/JmxFileUtil.java b/api-test/backend/src/main/java/io/metersphere/api/jmeter/utils/JmxFileUtil.java new file mode 100644 index 0000000000..3205d12f52 --- /dev/null +++ b/api-test/backend/src/main/java/io/metersphere/api/jmeter/utils/JmxFileUtil.java @@ -0,0 +1,75 @@ +package io.metersphere.api.jmeter.utils; + +import io.metersphere.commons.utils.JSONUtil; +import io.metersphere.dto.AttachmentBodyFile; +import io.metersphere.utils.LoggerUtil; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JmxFileUtil { + public static final String REDIS_JMX_FILE_PREFIX = "JMX.FILE."; + + private static final String JMX_INFO_KEY_ID = "id"; + private static final String JMX_INFO_KEY_FILE_PATH = "filePath"; + + private static final String JMX_INFO_KEY_FILE_NAME = "fileName"; + + private static final String JMX_INFO_KEY_FILE_STORAGE = "storage"; + + public static String getRedisJmxFileString(List listFile) { + List> jmxFileList = new ArrayList<>(); + listFile.forEach(file -> { + Map fileInfoMap = new HashMap<>(); + if (StringUtils.isNotBlank(file.getFileMetadataId())) { + fileInfoMap.put(JMX_INFO_KEY_ID, file.getFileMetadataId()); + } + if (StringUtils.isNotBlank(file.getFilePath())) { + fileInfoMap.put(JMX_INFO_KEY_FILE_PATH, file.getFilePath()); + } + if (StringUtils.isNotBlank(file.getName())) { + fileInfoMap.put(JMX_INFO_KEY_FILE_NAME, file.getName()); + } + if (StringUtils.isNotBlank(file.getFileStorage())) { + fileInfoMap.put(JMX_INFO_KEY_FILE_STORAGE, file.getFileStorage()); + } + + if (MapUtils.isNotEmpty(fileInfoMap)) { + jmxFileList.add(fileInfoMap); + } + }); + return JSONUtil.toJSONString(jmxFileList); + } + + public static List formatRedisJmxFileString(Object attachmentFileObj) { + List returnList = new ArrayList<>(); + try { + if (attachmentFileObj != null) { + List list = JSONUtil.parseArray(attachmentFileObj.toString(), Map.class); + list.forEach(itemMap -> { + AttachmentBodyFile file = new AttachmentBodyFile(); + if (itemMap.containsKey(JMX_INFO_KEY_ID)) { + file.setFileMetadataId(itemMap.get(JMX_INFO_KEY_ID).toString()); + } + if (itemMap.containsKey(JMX_INFO_KEY_FILE_PATH)) { + file.setFilePath(itemMap.get(JMX_INFO_KEY_FILE_PATH).toString()); + } + if (itemMap.containsKey(JMX_INFO_KEY_FILE_NAME)) { + file.setName(itemMap.get(JMX_INFO_KEY_FILE_NAME).toString()); + } + if (itemMap.containsKey(JMX_INFO_KEY_FILE_STORAGE)) { + file.setFileStorage(itemMap.get(JMX_INFO_KEY_FILE_STORAGE).toString()); + } + returnList.add(file); + }); + } + } catch (Exception e) { + LoggerUtil.error(e); + } + return returnList; + } +} diff --git a/api-test/backend/src/main/java/io/metersphere/commons/config/UtilsConfig.java b/api-test/backend/src/main/java/io/metersphere/commons/config/UtilsConfig.java new file mode 100644 index 0000000000..271cd8784e --- /dev/null +++ b/api-test/backend/src/main/java/io/metersphere/commons/config/UtilsConfig.java @@ -0,0 +1,13 @@ +package io.metersphere.commons.config; + +import io.metersphere.utils.TemporaryFileUtil; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class UtilsConfig { + @Bean + public TemporaryFileUtil temporaryFileUtil() { + return new TemporaryFileUtil(TemporaryFileUtil.MS_FILE_FOLDER); + } +} diff --git a/api-test/backend/src/main/java/io/metersphere/commons/constants/ElementConstants.java b/api-test/backend/src/main/java/io/metersphere/commons/constants/ElementConstants.java index e06aa00f22..a69764f558 100644 --- a/api-test/backend/src/main/java/io/metersphere/commons/constants/ElementConstants.java +++ b/api-test/backend/src/main/java/io/metersphere/commons/constants/ElementConstants.java @@ -54,6 +54,7 @@ public class ElementConstants { public static final String BEANSHELL = "beanshell"; public static final String IS_REF = "isRef"; public static final String FILE_ID = "fileId"; + public static final String RESOURCE_ID = "resourceId"; public static final String FILENAME = "filename"; public static final String COVER = "COVER"; diff --git a/api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java b/api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java index ff9db4156e..8f7aff8ffd 100644 --- a/api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java +++ b/api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java @@ -1,9 +1,12 @@ package io.metersphere.commons.utils; import io.metersphere.base.domain.FileMetadata; +import io.metersphere.base.domain.FileMetadataWithBLOBs; import io.metersphere.commons.constants.ElementConstants; import io.metersphere.commons.constants.StorageConstants; +import io.metersphere.dto.AttachmentBodyFile; import io.metersphere.dto.FileInfoDTO; +import io.metersphere.enums.JmxFileMetadataColumns; import io.metersphere.metadata.service.FileManagerService; import io.metersphere.metadata.service.FileMetadataService; import io.metersphere.metadata.vo.FileRequest; @@ -15,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.jmeter.config.CSVDataSet; import org.apache.jmeter.protocol.http.sampler.HTTPSamplerProxy; import org.apache.jmeter.protocol.http.util.HTTPFileArg; +import org.apache.jmeter.testelement.TestElement; import org.apache.jorphan.collections.HashTree; import org.springframework.web.multipart.MultipartFile; @@ -106,6 +110,183 @@ public class ApiFileUtil extends FileUtils { } } + public static void formatFilePathForNode(HashTree tree, String reportId, List fileList) { + if (tree != null) { + if (fileMetadataService == null) { + fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class); + } + + for (Object key : tree.keySet()) { + if (key == null) { + continue; + } + HashTree node = tree.get(key); + if (key instanceof HTTPSamplerProxy) { + getAttachmentBodyFileByHttp(key, reportId, fileList); + } else if (key instanceof CSVDataSet) { + getAttachmentBodyFileByCsv(key, reportId, fileList); + } + if (node != null) { + formatFilePathForNode(node, reportId, fileList); + } + } + } + } + + public static void getAttachmentBodyFileByCsv(Object tree, String reportId, List bodyFileList) { + CSVDataSet source = (CSVDataSet) tree; + if (StringUtils.isNotEmpty(source.getPropertyAsString(ElementConstants.FILENAME))) { + getAttachmentFileByTestElement(source, reportId, bodyFileList); + } + } + + public static void getAttachmentBodyFileByHttp(Object testElement, String reportId, List fileList) { + if (testElement == null) { + return; + } + HTTPSamplerProxy source = (HTTPSamplerProxy) testElement; + for (HTTPFileArg httpFileArg : source.getHTTPFiles()) { + getAttachmentFileByTestElement(httpFileArg, reportId, fileList); + } + } + + private static void getAttachmentFileByTestElement(TestElement testElement, String reportId, List bodyFileList) { + if (testElement == null) { + return; + } + String defaultFileName = null; + if (testElement instanceof HTTPFileArg) { + defaultFileName = ((HTTPFileArg) testElement).getPath(); + } else { + defaultFileName = testElement.getPropertyAsString(ElementConstants.FILENAME); + } + if (testElement.getPropertyAsBoolean(ElementConstants.IS_REF)) { + FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById( + testElement.getPropertyAsString(ElementConstants.FILE_ID)); + if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) { + + String path = getFilePathInJxm(reportId, fileMetadata.getName()); + + AttachmentBodyFile attachmentBodyFile = new AttachmentBodyFile(); + attachmentBodyFile.setFileMetadataId(fileMetadata.getId()); + attachmentBodyFile.setFileStorage(fileMetadata.getStorage()); + attachmentBodyFile.setName(fileMetadata.getName()); + attachmentBodyFile.setFileUpdateTime(fileMetadata.getUpdateTime()); + attachmentBodyFile.setProjectId(fileMetadata.getProjectId()); + attachmentBodyFile.setFilePath(path); + if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) { + attachmentBodyFile.setFileAttachInfoJson(fileMetadata.getAttachInfo()); + } + bodyFileList.add(attachmentBodyFile); + + testElement.setProperty(ElementConstants.FILENAME, path); + testElement.setProperty(JmxFileMetadataColumns.REF_FILE_STORAGE.name(), fileMetadata.getStorage()); + testElement.setProperty(JmxFileMetadataColumns.REF_FILE_NAME.name(), fileMetadata.getName()); + testElement.setProperty(JmxFileMetadataColumns.REF_FILE_UPDATE_TIME.name(), fileMetadata.getUpdateTime()); + testElement.setProperty(JmxFileMetadataColumns.REF_FILE_PROJECT_ID.name(), fileMetadata.getProjectId()); + if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) { + testElement.setProperty(JmxFileMetadataColumns.REF_FILE_ATTACH_INFO.name(), fileMetadata.getAttachInfo()); + } + + if (testElement instanceof HTTPFileArg) { + ((HTTPFileArg) testElement).setPath(path); + } + + } + } else { + if (StringUtils.isNotBlank(defaultFileName) && new File(defaultFileName).exists()) { + //判断本地文件 + AttachmentBodyFile attachmentBodyFile = new AttachmentBodyFile(); + attachmentBodyFile.setFileStorage(StorageConstants.LOCAL.name()); + attachmentBodyFile.setName(defaultFileName); + attachmentBodyFile.setFilePath(defaultFileName); + bodyFileList.add(attachmentBodyFile); + } else if (StringUtils.isNotBlank(testElement.getPropertyAsString(ElementConstants.RESOURCE_ID))) { + // 从MinIO下载 + AttachmentBodyFile attachmentBodyFile = new AttachmentBodyFile(); + attachmentBodyFile.setFileStorage(StorageConstants.MINIO.name()); + attachmentBodyFile.setName(testElement.getPropertyAsString(ElementConstants.RESOURCE_ID)); + attachmentBodyFile.setFilePath(defaultFileName); + bodyFileList.add(attachmentBodyFile); + } + } + } + + + private static String getFilePathInJxm(String reportId, String fileName) { + return StringUtils.join(BODY_FILE_DIR, File.separator, reportId, File.separator, fileName); + } + + // public static void formatFilePathForNode(HashTree tree, String reportId, List fileList) { + // if (fileMetadataService == null) { + // fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class); + // } + // for (Object key : tree.keySet()) { + // if (key == null) { + // continue; + // } + // HashTree node = tree.get(key); + // if (key instanceof HTTPSamplerProxy) { + // formatHttpFilePathForNode(key, reportId, fileList); + // } else if (key instanceof CSVDataSet) { + // formatCsvFilePathForNode(key, reportId, fileList); + // } + // if (node != null) { + // formatFilePathForNode(node, reportId, fileList); + // } + // } + // } + + // private static void formatCsvFilePathForNode(Object key, String reportId) { + // CSVDataSet source = (CSVDataSet) key; + // if (StringUtils.isNotEmpty(source.getPropertyAsString(ElementConstants.FILENAME))) { + // if (source.getPropertyAsBoolean(ElementConstants.IS_REF)) { + // FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById( + // source.getPropertyAsString(ElementConstants.FILE_ID)); + // if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) { + // String path = getFilePathInJxm(reportId, fileMetadata.getName()); + // ((CSVDataSet) key).setProperty(ElementConstants.FILENAME, path); + // ((CSVDataSet) key).setProperty(ElementConstants.FILE_STORAGE, fileMetadata.getStorage()); + // ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_NAME, fileMetadata.getName()); + // ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_UPDATE_TIME, fileMetadata.getUpdateTime()); + // ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_PROJECT_ID, fileMetadata.getProjectId()); + // if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) { + // ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_ATTACH_INFO, fileMetadata.getAttachInfo()); + // } + // } + // } + // } + // } + + // private static void formatHttpFilePathForNode(Object key, String reportId, List fileList) { + // if (key == null) { + // return; + // } + // HTTPSamplerProxy source = (HTTPSamplerProxy) key; + // for (HTTPFileArg httpFileArg : source.getHTTPFiles()) { + // if (httpFileArg.getPropertyAsBoolean(ElementConstants.IS_REF)) { + // FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById( + // httpFileArg.getPropertyAsString(ElementConstants.FILE_ID)); + // if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) { + // String path = getFilePathInJxm(reportId, fileMetadata.getName()); + // httpFileArg.setPath(path); + // httpFileArg.setProperty(ElementConstants.FILE_STORAGE, fileMetadata.getStorage()); + // httpFileArg.setProperty(ElementConstants.REF_FILE_NAME, fileMetadata.getName()); + // httpFileArg.setProperty(ElementConstants.REF_FILE_UPDATE_TIME, fileMetadata.getUpdateTime()); + // httpFileArg.setProperty(ElementConstants.REF_FILE_PROJECT_ID, fileMetadata.getProjectId()); + // if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) { + // httpFileArg.setProperty(ElementConstants.REF_FILE_ATTACH_INFO, fileMetadata.getAttachInfo()); + // } + // + // fileList.add(new AttachmentBodyFile() {{ + // this.setFileMetadataId(fileMetadata.getId()); + // }}); + // } + // } + // } + // + // + /** * 获取当前jmx 涉及到的文件 执行时 * @@ -141,19 +322,12 @@ public class ApiFileUtil extends FileUtils { if (source.getPropertyAsBoolean(ElementConstants.IS_REF)) { FileMetadata fileMetadata = fileMetadataService.getFileMetadataById( source.getPropertyAsString(ElementConstants.FILE_ID)); - if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) { file.setStorage(fileMetadata.getStorage()); file.setFileId(source.getPropertyAsString(ElementConstants.FILE_ID)); String fileName = StringUtils.join(reportId, File.separator, fileMetadata.getName()); file.setName(fileName); - - String path = StringUtils.join( - BODY_FILE_DIR, - File.separator, - reportId, - File.separator, - fileMetadata.getName()); + String path = getFilePathInJxm(reportId, fileMetadata.getName()); ((CSVDataSet) key).setProperty(ElementConstants.FILENAME, path); } } else if (!new File(source.getPropertyAsString(ElementConstants.FILENAME)).exists() @@ -184,13 +358,7 @@ public class ApiFileUtil extends FileUtils { file.setStorage(fileMetadata.getStorage()); file.setFileId(httpFileArg.getPropertyAsString(ElementConstants.FILE_ID)); file.setName(reportId + File.separator + fileMetadata.getName()); - String path = StringUtils.join( - BODY_FILE_DIR, - File.separator, - reportId, - File.separator, - fileMetadata.getName()); - + String path = getFilePathInJxm(reportId, fileMetadata.getName()); httpFileArg.setPath(path); } } else if (!new File(httpFileArg.getPath()).exists() diff --git a/api-test/backend/src/main/java/io/metersphere/commons/utils/HashTreeUtil.java b/api-test/backend/src/main/java/io/metersphere/commons/utils/HashTreeUtil.java index e6d9b235ea..59eb4e5e7b 100644 --- a/api-test/backend/src/main/java/io/metersphere/commons/utils/HashTreeUtil.java +++ b/api-test/backend/src/main/java/io/metersphere/commons/utils/HashTreeUtil.java @@ -251,7 +251,7 @@ public class HashTreeUtil { FileMetadataService fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class); if (fileMetadataService != null) { Map multipartFiles = new LinkedHashMap<>(); - List fileInfoDTOList = fileMetadataService.downloadFileByIds(repositoryFileMap.keySet()); + List fileInfoDTOList = fileMetadataService.downloadApiExecuteFilesByIds(repositoryFileMap.keySet()); fileInfoDTOList.forEach(dto -> { if (ArrayUtils.isNotEmpty(dto.getFileByte())) { String key = StringUtils.join( diff --git a/api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java b/api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java index 5e2fee0acb..5a65d064c4 100644 --- a/api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java +++ b/api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java @@ -66,6 +66,7 @@ public class ApiJMeterFileController { .body(bytes); } + @PostMapping("download/files") public ResponseEntity downloadJmeterFiles(@RequestBody BodyFileRequest request) { byte[] bytes = apiJmeterFileService.zipFilesToByteArray(request); diff --git a/api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java b/api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java index 05844197e4..06e6e6cc85 100644 --- a/api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java +++ b/api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java @@ -5,6 +5,7 @@ import io.metersphere.api.dto.EnvironmentType; import io.metersphere.api.dto.definition.request.ElementUtil; import io.metersphere.api.dto.definition.request.MsTestPlan; import io.metersphere.api.exec.api.ApiCaseSerialService; +import io.metersphere.api.jmeter.utils.JmxFileUtil; import io.metersphere.base.domain.*; import io.metersphere.base.mapper.ApiExecutionQueueDetailMapper; import io.metersphere.base.mapper.ApiScenarioMapper; @@ -13,6 +14,7 @@ import io.metersphere.base.mapper.plan.TestPlanApiScenarioMapper; import io.metersphere.commons.constants.ApiRunMode; import io.metersphere.commons.constants.PluginScenario; import io.metersphere.commons.utils.*; +import io.metersphere.dto.AttachmentBodyFile; import io.metersphere.dto.JmeterRunRequestDTO; import io.metersphere.environment.service.BaseEnvGroupProjectService; import io.metersphere.metadata.service.FileMetadataService; @@ -24,6 +26,7 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.jorphan.collections.HashTree; +import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; import java.io.ByteArrayOutputStream; @@ -50,6 +53,8 @@ public class ApiJMeterFileService { private FileMetadataService fileMetadataService; @Resource private PluginMapper pluginMapper; + @Resource + private RedisTemplate redisTemplate; // 接口测试 用例/接口 private static final List CASE_MODES = new ArrayList<>() {{ @@ -132,7 +137,7 @@ public class ApiJMeterFileService { if (hashTree != null) { ElementUtil.coverArguments(hashTree); } - return zipFilesToByteArray((reportId + "_" + remoteTestId), hashTree); + return zipFilesToByteArray((reportId + "_" + remoteTestId), reportId, hashTree); } public byte[] downloadJmeterJar() { @@ -221,15 +226,6 @@ public class ApiJMeterFileService { return jarFiles; } - private Map getMultipartFiles(String reportId, HashTree hashTree) { - Map multipartFiles = new LinkedHashMap<>(); - // 获取附件 - List files = new LinkedList<>(); - ApiFileUtil.getExecuteFiles(hashTree, reportId, files); - HashTreeUtil.downFile(files, multipartFiles, fileMetadataService); - return multipartFiles; - } - private String replaceJmx(String jmx) { jmx = StringUtils.replace(jmx, "", ""); @@ -239,12 +235,25 @@ public class ApiJMeterFileService { return jmx; } - private byte[] zipFilesToByteArray(String testId, HashTree hashTree) { - String bodyFilePath = FileUtils.BODY_FILE_DIR; + private byte[] zipFilesToByteArray(String testId, String reportId, HashTree hashTree) { String fileName = testId + ".jmx"; - // 获取JMX使用到的附件 - Map multipartFiles = this.getMultipartFiles(testId, hashTree); + /* + v2.7版本修改jmx附件逻辑： + 在node执行机器设置临时文件目录。这里只提供jmx + node拿到jmx后解析jmx中包含的附件信息。附件通过以下流程来获取： + 1 从node执行机的临时文件夹 + 2 临时文件夹中未找到的文件，根据文件类型来判断是否从minio/git下载，然后缓存到临时文件夹。 + 3 MinIO、Git中依然未能找到的文件（主要是Local文件），通过主工程下载，然后缓存到临时文件夹 + 所以这里不再使用以下方法来获取附件内容 + Map multipartFiles = this.getMultipartFiles(testId, hashTree); + 转为解析jmx中附件节点，赋予相关信息(例如文件关联类型、路径、更新时间等),并将文件信息存储在redis中，为了进行连接ms下载时的安全校验 + */ + List attachmentBodyFileList = new ArrayList<>(); + ApiFileUtil.formatFilePathForNode(hashTree, testId, attachmentBodyFileList); + if (CollectionUtils.isNotEmpty(attachmentBodyFileList)) { + redisTemplate.opsForValue().set(JmxFileUtil.REDIS_JMX_FILE_PREFIX + reportId, JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList)); + } String jmx = new MsTestPlan().getJmx(hashTree); // 处理dubbo请求生成jmx文件 @@ -255,17 +264,6 @@ public class ApiJMeterFileService { // 每个测试生成一个文件夹 files.put(fileName, jmx.getBytes(StandardCharsets.UTF_8)); - if (multipartFiles != null && !multipartFiles.isEmpty()) { - for (String k : multipartFiles.keySet()) { - byte[] v = multipartFiles.get(k); - if (k.startsWith(bodyFilePath)) { - files.put(StringUtils.substringAfter(k, bodyFilePath), v); - } else { - LogUtil.error("WARNING:Attachment path is not in body_file_path: " + k); - files.put(k, v); - } - } - } return listBytesToZip(files); } @@ -290,10 +288,11 @@ public class ApiJMeterFileService { public byte[] zipFilesToByteArray(BodyFileRequest request) { Map files = new LinkedHashMap<>(); if (CollectionUtils.isNotEmpty(request.getBodyFiles())) { + List bodyFiles = this.getLegalFiles(request); LoggerUtil.info("开始从三方仓库下载文件"); - HashTreeUtil.downFile(request.getBodyFiles(), files, fileMetadataService); + HashTreeUtil.downFile(bodyFiles, files, fileMetadataService); LoggerUtil.info("从三方仓库下载文件"); - for (BodyFile bodyFile : request.getBodyFiles()) { + for (BodyFile bodyFile : bodyFiles) { File file = new File(bodyFile.getName()); if (!file.exists()) { // 从MinIO下载 @@ -308,7 +307,40 @@ public class ApiJMeterFileService { } } } - return listBytesToZip(files); + Map zipFiles = new LinkedHashMap<>(); + for (Map.Entry entry : files.entrySet()) { + //去除文件路径前的body路径 + String filePath = entry.getKey(); + if (StringUtils.startsWith(filePath, FileUtils.BODY_FILE_DIR + "/")) { + filePath = StringUtils.substring(filePath, FileUtils.BODY_FILE_DIR.length() + 1); + } + zipFiles.put(filePath, entry.getValue()); + } + return listBytesToZip(zipFiles); + } + + private List getLegalFiles(BodyFileRequest request) { + List returnList = new ArrayList<>(); + + Object jmxFileInfoObj = redisTemplate.opsForValue().get(JmxFileUtil.REDIS_JMX_FILE_PREFIX + request.getReportId()); + List fileInJmx = JmxFileUtil.formatRedisJmxFileString(jmxFileInfoObj); + if (CollectionUtils.isNotEmpty(request.getBodyFiles())) { + request.getBodyFiles().forEach(attachmentBodyFile -> { + for (AttachmentBodyFile jmxFile : fileInJmx) { + if (StringUtils.isBlank(attachmentBodyFile.getRefResourceId())) { + if (StringUtils.equals(attachmentBodyFile.getName(), jmxFile.getFilePath()) + && StringUtils.equals(attachmentBodyFile.getName(), jmxFile.getName())) { + returnList.add(attachmentBodyFile); + } + } else { + if (StringUtils.equals(attachmentBodyFile.getRefResourceId(), jmxFile.getFileMetadataId())) { + returnList.add(attachmentBodyFile); + } + } + } + }); + } + return returnList; } } diff --git a/framework/sdk-parent/jmeter/src/main/java/io/metersphere/dto/AttachmentBodyFile.java b/framework/sdk-parent/jmeter/src/main/java/io/metersphere/dto/AttachmentBodyFile.java new file mode 100644 index 0000000000..b6e601875c --- /dev/null +++ b/framework/sdk-parent/jmeter/src/main/java/io/metersphere/dto/AttachmentBodyFile.java @@ -0,0 +1,29 @@ +package io.metersphere.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class AttachmentBodyFile { + private String id; + private String name; + // 调试附件处理 + private String refResourceId; + + //JMX中的路径 + private String filePath; + + //在文件服务器里的路径 + private String remotePath; + + //在本地节点的路径 + private String localPath; + private boolean isRef; + private String fileMetadataId; + private String projectId; + private long FileUpdateTime; + private String fileStorage; + private String fileAttachInfoJson; + private byte[] fileBytes; +} diff --git a/framework/sdk-parent/jmeter/src/main/java/io/metersphere/enums/JmxFileMetadataColumns.java b/framework/sdk-parent/jmeter/src/main/java/io/metersphere/enums/JmxFileMetadataColumns.java new file mode 100644 index 0000000000..485848a7a4 --- /dev/null +++ b/framework/sdk-parent/jmeter/src/main/java/io/metersphere/enums/JmxFileMetadataColumns.java @@ -0,0 +1,9 @@ +package io.metersphere.enums; + +public enum JmxFileMetadataColumns { + REF_FILE_STORAGE, + REF_FILE_UPDATE_TIME, + REF_FILE_PROJECT_ID, + REF_FILE_ATTACH_INFO, + REF_FILE_NAME +} diff --git a/framework/sdk-parent/jmeter/src/main/java/io/metersphere/utils/TemporaryFileUtil.java b/framework/sdk-parent/jmeter/src/main/java/io/metersphere/utils/TemporaryFileUtil.java new file mode 100644 index 0000000000..d57625b93a --- /dev/null +++ b/framework/sdk-parent/jmeter/src/main/java/io/metersphere/utils/TemporaryFileUtil.java @@ -0,0 +1,130 @@ +package io.metersphere.utils; + +import org.apache.commons.lang3.StringUtils; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +public class TemporaryFileUtil { + public final String fileFolder; + + public static final String NODE_FILE_FOLDER = "node"; + public static final String MS_FILE_FOLDER = "ms"; + + public static final String DEFAULT_FILE_FOLDER = "default"; + + public TemporaryFileUtil(String folder) { + if (StringUtils.isBlank(folder)) { + folder = DEFAULT_FILE_FOLDER; + } + this.fileFolder = File.separator + "opt" + + File.separator + "metersphere" + + File.separator + "data" + + File.separator + "body" + + File.separator + "local-file" + + File.separator + folder + + File.separator; + } + + public String generateFileDir(String folder) { + if (StringUtils.isBlank(folder)) { + folder = DEFAULT_FILE_FOLDER; + } + return fileFolder + folder + File.separator; + } + + public String generateFilePath(String folder, long updateTime, String fileName) { + String finalFileName = updateTime > 0 ? updateTime + "_" + fileName : fileName; + return generateFileDir(folder) + finalFileName; + } + + public File getFile(String folder, long updateTime, String fileName) { + File file = new File(generateFilePath(folder, updateTime, fileName)); + if (file.exists()) { + return file; + } else { + return null; + } + } + + public void saveFile(String folder, long updateTime, String fileName, byte[] fileBytes) { + //删除过期文件 + deleteOldFile(folder, fileName); + this.createFile(generateFilePath(folder, updateTime, fileName), fileBytes); + } + + public void saveFileByParamCheck(String folder, long updateTime, String fileName, byte[] fileBytes) { + if (fileBytes != null && StringUtils.isNotBlank(folder) && updateTime > 0 + && StringUtils.isNotBlank(fileName) && fileBytes.length > 0) { + //删除过期文件 + deleteOldFile(folder, fileName); + this.createFile(generateFilePath(folder, updateTime, fileName), fileBytes); + } + } + + private void deleteOldFile(String folder, String deleteFileName) { + List deleteFileList = new ArrayList<>(); + File file = new File(generateFileDir(folder)); + if (file.exists() && file.isDirectory()) { + String[] fileNameArr = file.list(); + if (fileNameArr != null) { + for (String fileName : fileNameArr) { + if (fileName.endsWith("_" + deleteFileName)) { + deleteFileList.add(fileName); + } + } + } + } + deleteFileList.forEach(fileName -> this.deleteFile(generateFileDir(folder) + fileName)); + } + + public byte[] fileToByte(File tradeFile) { + byte[] buffer = null; + try (FileInputStream fis = new FileInputStream(tradeFile); + ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + byte[] b = new byte[1024]; + int n; + while ((n = fis.read(b)) != -1) { + bos.write(b, 0, n); + } + buffer = bos.toByteArray(); + } catch (Exception e) { + LoggerUtil.error(e); + } + return buffer; + } + + private void createFile(String filePath, byte[] fileBytes) { + File file = new File(filePath); + if (file.exists()) { + this.deleteFile(filePath); + } + try { + File dir = file.getParentFile(); + if (!dir.exists()) { + dir.mkdirs(); + } + file.createNewFile(); + } catch (Exception e) { + LoggerUtil.error(e); + } + + try (InputStream in = new ByteArrayInputStream(fileBytes); OutputStream out = new FileOutputStream(file)) { + final int MAX = 4096; + byte[] buf = new byte[MAX]; + for (int bytesRead = in.read(buf, 0, MAX); bytesRead != -1; bytesRead = in.read(buf, 0, MAX)) { + out.write(buf, 0, bytesRead); + } + } catch (IOException e) { + LoggerUtil.error(e); + } + } + + public void deleteFile(String path) { + File file = new File(path); + if (file.exists()) { + file.delete(); + } + } +} diff --git a/framework/sdk-parent/sdk/src/main/java/io/metersphere/dto/FileInfoDTO.java b/framework/sdk-parent/sdk/src/main/java/io/metersphere/dto/FileInfoDTO.java index 3970c606f0..a81eaae984 100644 --- a/framework/sdk-parent/sdk/src/main/java/io/metersphere/dto/FileInfoDTO.java +++ b/framework/sdk-parent/sdk/src/main/java/io/metersphere/dto/FileInfoDTO.java @@ -8,6 +8,8 @@ import lombok.Data; public class FileInfoDTO { private String id; private String fileName; + private String projectId; + private long fileLastUpdateTime; private String storage; private String path; private byte[] fileByte; diff --git a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/GitFileRepository.java b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/GitFileRepository.java index 9ca8295932..1483c20cb5 100644 --- a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/GitFileRepository.java +++ b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/GitFileRepository.java @@ -35,14 +35,14 @@ public class GitFileRepository implements FileRepository { @Override public byte[] getFile(FileRequest request) throws Exception { - byte[] buffer = new byte[0]; + byte[] fileBytes = new byte[0]; if (request.getFileAttachInfo() != null) { RemoteFileAttachInfo gitFileInfo = request.getFileAttachInfo(); GitRepositoryUtil repositoryUtils = new GitRepositoryUtil( gitFileInfo.getRepositoryPath(), gitFileInfo.getUserName(), gitFileInfo.getToken()); - buffer = repositoryUtils.getSingleFile(gitFileInfo.getFilePath(), gitFileInfo.getCommitId()); + fileBytes = repositoryUtils.getSingleFile(gitFileInfo.getFilePath(), gitFileInfo.getCommitId()); } - return buffer; + return fileBytes; } @Override @@ -71,32 +71,41 @@ public class GitFileRepository implements FileRepository { List requestList = entry.getValue(); RemoteFileAttachInfo baseGitFileInfo = null; - List repositoryRequestList = new ArrayList<>(); + List downloadFileList = new ArrayList<>(); + Map fileByteMap = new HashMap<>(); for (FileRequest fileRequest : requestList) { RemoteFileAttachInfo gitFileInfo = fileRequest.getFileAttachInfo(); if (baseGitFileInfo == null) { baseGitFileInfo = gitFileInfo; } - repositoryRequestList.add(new RepositoryRequest() {{ + downloadFileList.add(new RepositoryRequest() {{ this.setCommitId(gitFileInfo.getCommitId()); this.setFilePath(gitFileInfo.getFilePath()); this.setFileMetadataId(fileRequest.getResourceId()); + this.setProjectId(fileRequest.getProjectId()); + this.setUpdateTime(fileRequest.getUpdateTime()); }}); } - GitRepositoryUtil repositoryUtils = new GitRepositoryUtil( - baseGitFileInfo.getRepositoryPath(), - baseGitFileInfo.getUserName(), baseGitFileInfo.getToken()); + if (CollectionUtils.isNotEmpty(downloadFileList) && baseGitFileInfo != null) { + GitRepositoryUtil repositoryUtils = new GitRepositoryUtil( + baseGitFileInfo.getRepositoryPath(), + baseGitFileInfo.getUserName(), baseGitFileInfo.getToken()); + Map downloadFileMap = repositoryUtils.getFiles(downloadFileList); + fileByteMap.putAll(downloadFileMap); + } - Map fileByteMap = repositoryUtils.getFiles(repositoryRequestList); - repositoryRequestList.forEach(repositoryFile -> { - if (fileByteMap.get(repositoryFile.getFileMetadataId()) != null) { + requestList.forEach(fileRequest -> { + RemoteFileAttachInfo gitFileInfo = fileRequest.getFileAttachInfo(); + if (fileByteMap.get(fileRequest.getResourceId()) != null) { FileInfoDTO repositoryFileDTO = new FileInfoDTO( - repositoryFile.getFileMetadataId(), - MetadataUtils.getFileNameByRemotePath(repositoryFile.getFilePath()), + fileRequest.getResourceId(), + MetadataUtils.getFileNameByRemotePath(gitFileInfo.getFilePath()), + fileRequest.getProjectId(), + fileRequest.getUpdateTime(), StorageConstants.GIT.name(), - repositoryFile.getFilePath(), - fileByteMap.get(repositoryFile.getFileMetadataId())); + gitFileInfo.getFilePath(), + fileByteMap.get(fileRequest.getResourceId())); list.add(repositoryFileDTO); } }); diff --git a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/LocalFileRepository.java b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/LocalFileRepository.java index c02092bff2..b3cff109a3 100644 --- a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/LocalFileRepository.java +++ b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/LocalFileRepository.java @@ -119,6 +119,7 @@ public class LocalFileRepository implements FileRepository { } return new FileInfoDTO( fileRequest.getResourceId(), fileRequest.getFileName(), + fileRequest.getProjectId(), fileRequest.getUpdateTime(), fileRequest.getStorage(), fileRequest.getPath(), content); }).collect(Collectors.toList()); } diff --git a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/MinIOFileRepository.java b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/MinIOFileRepository.java index 664e8af765..43635e3a5f 100644 --- a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/MinIOFileRepository.java +++ b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/MinIOFileRepository.java @@ -7,12 +7,12 @@ import io.metersphere.dto.FileInfoDTO; import io.metersphere.metadata.vo.FileRequest; import io.minio.*; import io.minio.messages.Item; +import jakarta.annotation.Resource; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.springframework.web.multipart.MultipartFile; -import jakarta.annotation.Resource; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.ArrayList; @@ -133,7 +133,7 @@ public class MinIOFileRepository implements FileRepository { List list = new ArrayList<>(); if (CollectionUtils.isNotEmpty(requestList)) { for (FileRequest fileRequest : requestList) { - FileInfoDTO fileInfoDTO = new FileInfoDTO(fileRequest.getResourceId(), fileRequest.getFileName(), fileRequest.getStorage(), fileRequest.getPath(), this.getFile(fileRequest)); + FileInfoDTO fileInfoDTO = new FileInfoDTO(fileRequest.getResourceId(), fileRequest.getFileName(), fileRequest.getProjectId(), fileRequest.getUpdateTime(), fileRequest.getStorage(), fileRequest.getPath(), this.getFile(fileRequest)); list.add(fileInfoDTO); } } diff --git a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/service/FileMetadataService.java b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/service/FileMetadataService.java index 3fa3b84df9..c345ac7e63 100644 --- a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/service/FileMetadataService.java +++ b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/service/FileMetadataService.java @@ -10,10 +10,7 @@ import io.metersphere.commons.constants.ApiTestConstants; import io.metersphere.commons.constants.FileModuleTypeConstants; import io.metersphere.commons.constants.StorageConstants; import io.metersphere.commons.exception.MSException; -import io.metersphere.commons.utils.FileUtils; -import io.metersphere.commons.utils.JSON; -import io.metersphere.commons.utils.LogUtil; -import io.metersphere.commons.utils.SessionUtils; +import io.metersphere.commons.utils.*; import io.metersphere.dto.FileInfoDTO; import io.metersphere.i18n.Translator; import io.metersphere.log.utils.ReflexObjectUtil; @@ -25,6 +22,8 @@ import io.metersphere.metadata.utils.MetadataUtils; import io.metersphere.metadata.vo.*; import io.metersphere.request.OrderRequest; import io.metersphere.request.QueryProjectFileRequest; +import io.metersphere.utils.TemporaryFileUtil; +import jakarta.annotation.Resource; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpHeaders; @@ -33,7 +32,6 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import jakarta.annotation.Resource; import java.io.File; import java.io.InputStream; import java.util.*; @@ -54,6 +52,8 @@ public class FileMetadataService { @Resource private FileAssociationMapper fileAssociationMapper; + private TemporaryFileUtil temporaryFileUtil; + public List create(FileMetadataCreateRequest fileMetadata, List files) { List result = new ArrayList<>(); if (fileMetadata == null) { @@ -239,6 +239,7 @@ public class FileMetadataService { request.setPath(fileMetadata.getPath()); request.setStorage(fileMetadata.getStorage()); request.setFileAttachInfoByString(fileMetadata.getAttachInfo()); + request.setUpdateTime(fileMetadata.getUpdateTime()); bytes = fileManagerService.downloadFile(request); } return bytes; @@ -347,7 +348,7 @@ public class FileMetadataService { return fileMetadata; } - public FileMetadata getFileMetadataById(String fileId) { + public FileMetadataWithBLOBs getFileMetadataById(String fileId) { return fileMetadataMapper.selectByPrimaryKey(fileId); } @@ -520,8 +521,70 @@ public class FileMetadataService { return fileMetadataList.stream().map(FileMetadata::getId).collect(Collectors.toList()); } + /** + * 接口测试执行时下载附件的方法。 + * 该方法会优先判断是否存在已下载好的文件，避免多次执行造成多次下载的情况 + * + * @param fileIdList + * @return + */ + public List downloadApiExecuteFilesByIds(Collection fileIdList) { + if (temporaryFileUtil == null) { + temporaryFileUtil = CommonBeanFactory.getBean(TemporaryFileUtil.class); + } + + List fileInfoDTOList = new ArrayList<>(); + if (CollectionUtils.isEmpty(fileIdList)) { + return fileInfoDTOList; + } + LogUtil.info(JSON.toJSONString(fileIdList) + " 获取执行文件开始"); + FileMetadataExample example = new FileMetadataExample(); + example.createCriteria().andIdIn(new ArrayList<>(fileIdList)); + List fileMetadataWithBLOBList = fileMetadataMapper.selectByExampleWithBLOBs(example); + List downloadFileRequest = new ArrayList<>(); + //检查是否存在已下载的文件 + fileMetadataWithBLOBList.forEach(fileMetadata -> { + File file = temporaryFileUtil.getFile(fileMetadata.getProjectId(), fileMetadata.getUpdateTime(), fileMetadata.getName()); + if (file != null && file.exists() && file.isFile()) { + FileInfoDTO fileInfoDTO = new FileInfoDTO(fileMetadata.getId(), fileMetadata.getName(), fileMetadata.getProjectId(), fileMetadata.getUpdateTime(), fileMetadata.getStorage(), fileMetadata.getPath(), FileUtils.fileToByte(file)); + fileInfoDTOList.add(fileInfoDTO); + } else { + downloadFileRequest.add(this.genFileRequest(fileMetadata)); + } + }); + List repositoryFileDTOList = fileManagerService.downloadFileBatch(downloadFileRequest); + //将文件存储到执行文件目录中，避免多次执行时触发多次下载 + if (CollectionUtils.isNotEmpty(repositoryFileDTOList)) { + repositoryFileDTOList.forEach(repositoryFile -> temporaryFileUtil.saveFileByParamCheck(repositoryFile.getProjectId(), repositoryFile.getFileLastUpdateTime(), repositoryFile.getFileName(), repositoryFile.getFileByte())); + fileInfoDTOList.addAll(repositoryFileDTOList); + } + return fileInfoDTOList; + } + + private FileRequest genFileRequest(FileMetadataWithBLOBs fileMetadata) { + if (fileMetadata != null) { + FileRequest request = new FileRequest(fileMetadata.getProjectId(), fileMetadata.getName(), fileMetadata.getType()); + request.setResourceId(fileMetadata.getId()); + request.setResourceType(fileMetadata.getResourceType()); + request.setPath(fileMetadata.getPath()); + request.setStorage(fileMetadata.getStorage()); + request.setUpdateTime(fileMetadata.getUpdateTime()); + if (StringUtils.equals(fileMetadata.getStorage(), StorageConstants.GIT.name())) { + try { + RemoteFileAttachInfo gitFileInfo = JSON.parseObject(fileMetadata.getAttachInfo(), RemoteFileAttachInfo.class); + request.setFileAttachInfo(gitFileInfo); + } catch (Exception e) { + LogUtil.error("解析Git附加信息【" + fileMetadata.getAttachInfo() + "】失败!", e); + } + } + return request; + } else { + return new FileRequest(); + } + } + public List downloadFileByIds(Collection fileIdList) { - if (org.apache.commons.collections.CollectionUtils.isEmpty(fileIdList)) { + if (CollectionUtils.isEmpty(fileIdList)) { return new ArrayList<>(0); } LogUtil.info(JSON.toJSONString(fileIdList) + " 获取文件开始"); @@ -532,22 +595,8 @@ public class FileMetadataService { List requestList = new ArrayList<>(); fileMetadataWithBLOBs.forEach(fileMetadata -> { - FileRequest request = new FileRequest(fileMetadata.getProjectId(), fileMetadata.getName(), fileMetadata.getType()); - request.setResourceId(fileMetadata.getId()); - request.setResourceType(fileMetadata.getResourceType()); - request.setPath(fileMetadata.getPath()); - request.setStorage(fileMetadata.getStorage()); - if (StringUtils.equals(fileMetadata.getStorage(), StorageConstants.GIT.name())) { - try { - RemoteFileAttachInfo gitFileInfo = JSON.parseObject(fileMetadata.getAttachInfo(), RemoteFileAttachInfo.class); - request.setFileAttachInfo(gitFileInfo); - } catch (Exception e) { - LogUtil.error("解析Git附加信息【" + fileMetadata.getAttachInfo() + "】失败!", e); - } - } - requestList.add(request); + requestList.add(this.genFileRequest(fileMetadata)); }); - List repositoryFileDTOList = fileManagerService.downloadFileBatch(requestList); LogUtil.info(JSON.toJSONString(fileIdList) + " 获取文件结束。"); return repositoryFileDTOList; @@ -564,12 +613,12 @@ public class FileMetadataService { RemoteFileAttachInfo gitFileAttachInfo = repositoryUtils.selectLastCommitIdByBranch(baseAttachInfo.getBranch(), baseAttachInfo.getFilePath()); if (gitFileAttachInfo != null && !StringUtils.equals(gitFileAttachInfo.getCommitId(), baseAttachInfo.getCommitId())) { - //有新的commitId，更新filemetadata的版本 - long thistime = System.currentTimeMillis(); - FileMetadataWithBLOBs newMetadata = this.genOtherVersionFileMetadata(baseMetadata, thistime, gitFileAttachInfo); + //有新的commitId，更新fileMetadata的版本 + long thisTime = System.currentTimeMillis(); + FileMetadataWithBLOBs newMetadata = this.genOtherVersionFileMetadata(baseMetadata, thisTime, gitFileAttachInfo); fileMetadataMapper.insert(newMetadata); - baseMetadata.setUpdateTime(thistime); + baseMetadata.setUpdateTime(thisTime); baseMetadata.setLatest(false); baseMetadata.setUpdateUser(SessionUtils.getUserId()); fileMetadataMapper.updateByPrimaryKeySelective(baseMetadata); diff --git a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/FileRequest.java b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/FileRequest.java index ba0046a917..77af273bbc 100644 --- a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/FileRequest.java +++ b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/FileRequest.java @@ -21,6 +21,10 @@ public class FileRequest { //文件附属信息 private RemoteFileAttachInfo fileAttachInfo; + //文件更新时间。 用于在缓存文件中判断是否需要更新 + private long updateTime; + //缓存文件夹中通过moduleId作为内部文件结构 + public FileRequest() { } @@ -30,7 +34,7 @@ public class FileRequest { this.projectId = projectId; this.type = StringUtils.isNotEmpty(type) ? type.toLowerCase() : null; this.fileName = name; - if (StringUtils.isNotEmpty(this.type) && !name.endsWith(this.type)) { + if (!StringUtils.equalsIgnoreCase(this.storage, StorageConstants.MINIO.name()) && StringUtils.isNotEmpty(this.type) && !name.endsWith(this.type)) { this.fileName = StringUtils.join(name, ".", this.type); } } diff --git a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/RepositoryRequest.java b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/RepositoryRequest.java index 7d01a42b4f..59f9900143 100644 --- a/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/RepositoryRequest.java +++ b/framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/RepositoryRequest.java @@ -7,4 +7,6 @@ public class RepositoryRequest { private String fileMetadataId; private String filePath; private String commitId; + private String projectId; + private long updateTime; } diff --git a/project-management/frontend/src/business/menu/file/list/FileVersionList.vue b/project-management/frontend/src/business/menu/file/list/FileVersionList.vue index 7f084e0a02..8b5e137fe3 100644 --- a/project-management/frontend/src/business/menu/file/list/FileVersionList.vue +++ b/project-management/frontend/src/business/menu/file/list/FileVersionList.vue @@ -1,19 +1,35 @@ - +