test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(接口测试): 执行附件本地缓存，避免每次执行都下载文件性能开销比较大 

--story=1011099 --user=宋天阳 执行附件本地缓存，避免每次执行都下载文件性能开销比较大
https://www.tapd.cn/55049933/s/1332544
This commit is contained in:
song-tianyang 2023-02-03 17:18:28 +08:00 committed by 建国
parent fb7d1243fe
commit 543f0463ec
21 changed files with 685 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api-test/backend/src/main/java/io/metersphere/api/jmeter/KafkaListenerTask.java
View File

@ -2,6 +2,7 @@ package io.metersphere.api.jmeter;
import com.fasterxml.jackson.core.type.TypeReference;
import io.metersphere.api.exec.queue.PoolExecBlockingQueueUtil;
import io.metersphere.api.jmeter.utils.JmxFileUtil;
import io.metersphere.commons.constants.ApiRunMode;
import io.metersphere.commons.constants.ExtendedParameter;
import io.metersphere.commons.utils.JSON;
@ -14,6 +15,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.data.redis.core.RedisTemplate;
import java.util.*;
@ -22,6 +24,9 @@ public class KafkaListenerTask implements Runnable {
private ConsumerRecord<?, String> record;
private ApiExecutionQueueService apiExecutionQueueService;
private TestResultService testResultService;
private RedisTemplate<String, Object> redisTemplate;
private static final Map<String, String> RUN_MODE_MAP = new HashMap<String, String>() {{
this.put(ApiRunMode.SCHEDULE_API_PLAN.name(), "schedule-task");
this.put(ApiRunMode.JENKINS_API_PLAN.name(), "schedule-task");
@ -53,6 +58,10 @@ public class KafkaListenerTask implements Runnable {
if (dto == null) {
return;
}
if (redisTemplate != null) {
redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId());
}
if (dto.getArbitraryData() != null && dto.getArbitraryData().containsKey(ExtendedParameter.TEST_END)
&& (Boolean) dto.getArbitraryData().get(ExtendedParameter.TEST_END)) {
resultDTOS.add(dto);

15
api-test/backend/src/main/java/io/metersphere/api/jmeter/MsApiBackendListener.java
View File

@ -2,6 +2,7 @@ package io.metersphere.api.jmeter;
import io.metersphere.api.exec.queue.PoolExecBlockingQueueUtil;
import io.metersphere.api.jmeter.utils.JmxFileUtil;
import io.metersphere.api.jmeter.utils.ReportStatusUtil;
import io.metersphere.commons.constants.CommonConstants;
import io.metersphere.commons.utils.*;
@ -14,11 +15,13 @@ import io.metersphere.service.ApiExecutionQueueService;
import io.metersphere.service.TestResultService;
import io.metersphere.utils.LoggerUtil;
import io.metersphere.utils.RetryResultUtil;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.StringUtils;
import org.apache.jmeter.samplers.SampleResult;
import org.apache.jmeter.services.FileServer;
import org.apache.jmeter.visualizers.backend.AbstractBackendListenerClient;
import org.apache.jmeter.visualizers.backend.BackendListenerContext;
import org.springframework.data.redis.core.RedisTemplate;
import java.io.Serializable;
import java.util.LinkedHashMap;
@ -34,6 +37,9 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen
// 当前场景报告/用例结果状态
private ResultVO resultVO;
@Resource
private RedisTemplate<String, Object> redisTemplate;
/**
* 参数初始化方法
*/
@ -58,6 +64,10 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen
if (dto.isRetryEnable()) {
queues.addAll(sampleResults);
} else {
if (redisTemplate != null) {
redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId());
}
if (!StringUtils.equals(dto.getReportType(), RunModeConstants.SET_REPORT.toString())) {
dto.setConsole(FixedCapacityUtil.getJmeterLogger(getReportId(), false));
}
@ -74,6 +84,11 @@ public class MsApiBackendListener extends AbstractBackendListenerClient implemen
public void teardownTest(BackendListenerContext context) {
try {
LoggerUtil.info("进入TEST-END处理报告" + dto.getRunMode(), dto.getReportId());
if (redisTemplate != null) {
redisTemplate.delete(JmxFileUtil.REDIS_JMX_FILE_PREFIX + dto.getReportId());
}
super.teardownTest(context);
// 获取执行日志
if (!StringUtils.equals(dto.getReportType(), RunModeConstants.SET_REPORT.toString())) {

7
api-test/backend/src/main/java/io/metersphere/api/jmeter/MsKafkaListener.java
View File

@ -8,14 +8,15 @@ import io.metersphere.service.ApiExecutionQueueService;
import io.metersphere.service.TestResultService;
import io.metersphere.service.definition.ApiDefinitionEnvService;
import io.metersphere.utils.LoggerUtil;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import jakarta.annotation.Resource;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
@ -38,6 +39,9 @@ public class MsKafkaListener {
// 线程池所使用的缓冲队列大小
private final static int WORK_QUEUE_SIZE = 10000;
@Resource
private RedisTemplate<String, Object> redisTemplate;
private final ThreadPoolExecutor threadPool = new ThreadPoolExecutor(
CORE_POOL_SIZE,
MAX_POOL_SIZE,
@ -55,6 +59,7 @@ public class MsKafkaListener {
task.setApiExecutionQueueService(apiExecutionQueueService);
task.setTestResultService(testResultService);
task.setRecord(item);
task.setRedisTemplate(redisTemplate);
threadPool.execute(task);
});
JvmUtil.memoryInfo();

75
api-test/backend/src/main/java/io/metersphere/api/jmeter/utils/JmxFileUtil.java Normal file
View File

@ -0,0 +1,75 @@
package io.metersphere.api.jmeter.utils;
import io.metersphere.commons.utils.JSONUtil;
import io.metersphere.dto.AttachmentBodyFile;
import io.metersphere.utils.LoggerUtil;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class JmxFileUtil {
public static final String REDIS_JMX_FILE_PREFIX = "JMX.FILE.";
private static final String JMX_INFO_KEY_ID = "id";
private static final String JMX_INFO_KEY_FILE_PATH = "filePath";
private static final String JMX_INFO_KEY_FILE_NAME = "fileName";
private static final String JMX_INFO_KEY_FILE_STORAGE = "storage";
public static String getRedisJmxFileString(List<AttachmentBodyFile> listFile) {
List<Map<String, String>> jmxFileList = new ArrayList<>();
listFile.forEach(file -> {
Map<String, String> fileInfoMap = new HashMap<>();
if (StringUtils.isNotBlank(file.getFileMetadataId())) {
fileInfoMap.put(JMX_INFO_KEY_ID, file.getFileMetadataId());
}
if (StringUtils.isNotBlank(file.getFilePath())) {
fileInfoMap.put(JMX_INFO_KEY_FILE_PATH, file.getFilePath());
}
if (StringUtils.isNotBlank(file.getName())) {
fileInfoMap.put(JMX_INFO_KEY_FILE_NAME, file.getName());
}
if (StringUtils.isNotBlank(file.getFileStorage())) {
fileInfoMap.put(JMX_INFO_KEY_FILE_STORAGE, file.getFileStorage());
}
if (MapUtils.isNotEmpty(fileInfoMap)) {
jmxFileList.add(fileInfoMap);
}
});
return JSONUtil.toJSONString(jmxFileList);
}
public static List<AttachmentBodyFile> formatRedisJmxFileString(Object attachmentFileObj) {
List<AttachmentBodyFile> returnList = new ArrayList<>();
try {
if (attachmentFileObj != null) {
List<Map> list = JSONUtil.parseArray(attachmentFileObj.toString(), Map.class);
list.forEach(itemMap -> {
AttachmentBodyFile file = new AttachmentBodyFile();
if (itemMap.containsKey(JMX_INFO_KEY_ID)) {
file.setFileMetadataId(itemMap.get(JMX_INFO_KEY_ID).toString());
}
if (itemMap.containsKey(JMX_INFO_KEY_FILE_PATH)) {
file.setFilePath(itemMap.get(JMX_INFO_KEY_FILE_PATH).toString());
}
if (itemMap.containsKey(JMX_INFO_KEY_FILE_NAME)) {
file.setName(itemMap.get(JMX_INFO_KEY_FILE_NAME).toString());
}
if (itemMap.containsKey(JMX_INFO_KEY_FILE_STORAGE)) {
file.setFileStorage(itemMap.get(JMX_INFO_KEY_FILE_STORAGE).toString());
}
returnList.add(file);
});
}
} catch (Exception e) {
LoggerUtil.error(e);
}
return returnList;
}
}

13
api-test/backend/src/main/java/io/metersphere/commons/config/UtilsConfig.java Normal file
View File

@ -0,0 +1,13 @@
package io.metersphere.commons.config;
import io.metersphere.utils.TemporaryFileUtil;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class UtilsConfig {
@Bean
public TemporaryFileUtil temporaryFileUtil() {
return new TemporaryFileUtil(TemporaryFileUtil.MS_FILE_FOLDER);
}
}

1
api-test/backend/src/main/java/io/metersphere/commons/constants/ElementConstants.java
View File

@ -54,6 +54,7 @@ public class ElementConstants {
public static final String BEANSHELL = "beanshell";
public static final String IS_REF = "isRef";
public static final String FILE_ID = "fileId";
public static final String RESOURCE_ID = "resourceId";
public static final String FILENAME = "filename";
public static final String COVER = "COVER";

198
api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java
View File

@ -1,9 +1,12 @@
package io.metersphere.commons.utils;
import io.metersphere.base.domain.FileMetadata;
import io.metersphere.base.domain.FileMetadataWithBLOBs;
import io.metersphere.commons.constants.ElementConstants;
import io.metersphere.commons.constants.StorageConstants;
import io.metersphere.dto.AttachmentBodyFile;
import io.metersphere.dto.FileInfoDTO;
import io.metersphere.enums.JmxFileMetadataColumns;
import io.metersphere.metadata.service.FileManagerService;
import io.metersphere.metadata.service.FileMetadataService;
import io.metersphere.metadata.vo.FileRequest;
@ -15,6 +18,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.jmeter.config.CSVDataSet;
import org.apache.jmeter.protocol.http.sampler.HTTPSamplerProxy;
import org.apache.jmeter.protocol.http.util.HTTPFileArg;
import org.apache.jmeter.testelement.TestElement;
import org.apache.jorphan.collections.HashTree;
import org.springframework.web.multipart.MultipartFile;
@ -106,6 +110,183 @@ public class ApiFileUtil extends FileUtils {
}
}
public static void formatFilePathForNode(HashTree tree, String reportId, List<AttachmentBodyFile> fileList) {
if (tree != null) {
if (fileMetadataService == null) {
fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class);
}
for (Object key : tree.keySet()) {
if (key == null) {
continue;
}
HashTree node = tree.get(key);
if (key instanceof HTTPSamplerProxy) {
getAttachmentBodyFileByHttp(key, reportId, fileList);
} else if (key instanceof CSVDataSet) {
getAttachmentBodyFileByCsv(key, reportId, fileList);
}
if (node != null) {
formatFilePathForNode(node, reportId, fileList);
}
}
}
}
public static void getAttachmentBodyFileByCsv(Object tree, String reportId, List<AttachmentBodyFile> bodyFileList) {
CSVDataSet source = (CSVDataSet) tree;
if (StringUtils.isNotEmpty(source.getPropertyAsString(ElementConstants.FILENAME))) {
getAttachmentFileByTestElement(source, reportId, bodyFileList);
}
}
public static void getAttachmentBodyFileByHttp(Object testElement, String reportId, List<AttachmentBodyFile> fileList) {
if (testElement == null) {
return;
}
HTTPSamplerProxy source = (HTTPSamplerProxy) testElement;
for (HTTPFileArg httpFileArg : source.getHTTPFiles()) {
getAttachmentFileByTestElement(httpFileArg, reportId, fileList);
}
}
private static void getAttachmentFileByTestElement(TestElement testElement, String reportId, List<AttachmentBodyFile> bodyFileList) {
if (testElement == null) {
return;
}
String defaultFileName = null;
if (testElement instanceof HTTPFileArg) {
defaultFileName = ((HTTPFileArg) testElement).getPath();
} else {
defaultFileName = testElement.getPropertyAsString(ElementConstants.FILENAME);
}
if (testElement.getPropertyAsBoolean(ElementConstants.IS_REF)) {
FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById(
testElement.getPropertyAsString(ElementConstants.FILE_ID));
if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) {
String path = getFilePathInJxm(reportId, fileMetadata.getName());
AttachmentBodyFile attachmentBodyFile = new AttachmentBodyFile();
attachmentBodyFile.setFileMetadataId(fileMetadata.getId());
attachmentBodyFile.setFileStorage(fileMetadata.getStorage());
attachmentBodyFile.setName(fileMetadata.getName());
attachmentBodyFile.setFileUpdateTime(fileMetadata.getUpdateTime());
attachmentBodyFile.setProjectId(fileMetadata.getProjectId());
attachmentBodyFile.setFilePath(path);
if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) {
attachmentBodyFile.setFileAttachInfoJson(fileMetadata.getAttachInfo());
}
bodyFileList.add(attachmentBodyFile);
testElement.setProperty(ElementConstants.FILENAME, path);
testElement.setProperty(JmxFileMetadataColumns.REF_FILE_STORAGE.name(), fileMetadata.getStorage());
testElement.setProperty(JmxFileMetadataColumns.REF_FILE_NAME.name(), fileMetadata.getName());
testElement.setProperty(JmxFileMetadataColumns.REF_FILE_UPDATE_TIME.name(), fileMetadata.getUpdateTime());
testElement.setProperty(JmxFileMetadataColumns.REF_FILE_PROJECT_ID.name(), fileMetadata.getProjectId());
if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) {
testElement.setProperty(JmxFileMetadataColumns.REF_FILE_ATTACH_INFO.name(), fileMetadata.getAttachInfo());
}
if (testElement instanceof HTTPFileArg) {
((HTTPFileArg) testElement).setPath(path);
}
}
} else {
if (StringUtils.isNotBlank(defaultFileName) && new File(defaultFileName).exists()) {
//判断本地文件
AttachmentBodyFile attachmentBodyFile = new AttachmentBodyFile();
attachmentBodyFile.setFileStorage(StorageConstants.LOCAL.name());
attachmentBodyFile.setName(defaultFileName);
attachmentBodyFile.setFilePath(defaultFileName);
bodyFileList.add(attachmentBodyFile);
} else if (StringUtils.isNotBlank(testElement.getPropertyAsString(ElementConstants.RESOURCE_ID))) {
// 从MinIO下载
AttachmentBodyFile attachmentBodyFile = new AttachmentBodyFile();
attachmentBodyFile.setFileStorage(StorageConstants.MINIO.name());
attachmentBodyFile.setName(testElement.getPropertyAsString(ElementConstants.RESOURCE_ID));
attachmentBodyFile.setFilePath(defaultFileName);
bodyFileList.add(attachmentBodyFile);
}
}
}
private static String getFilePathInJxm(String reportId, String fileName) {
return StringUtils.join(BODY_FILE_DIR, File.separator, reportId, File.separator, fileName);
}
// public static void formatFilePathForNode(HashTree tree, String reportId, List<AttachmentBodyFile> fileList) {
// if (fileMetadataService == null) {
// fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class);
// }
// for (Object key : tree.keySet()) {
// if (key == null) {
// continue;
// }
// HashTree node = tree.get(key);
// if (key instanceof HTTPSamplerProxy) {
// formatHttpFilePathForNode(key, reportId, fileList);
// } else if (key instanceof CSVDataSet) {
// formatCsvFilePathForNode(key, reportId, fileList);
// }
// if (node != null) {
// formatFilePathForNode(node, reportId, fileList);
// }
// }
// }
// private static void formatCsvFilePathForNode(Object key, String reportId) {
// CSVDataSet source = (CSVDataSet) key;
// if (StringUtils.isNotEmpty(source.getPropertyAsString(ElementConstants.FILENAME))) {
// if (source.getPropertyAsBoolean(ElementConstants.IS_REF)) {
// FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById(
// source.getPropertyAsString(ElementConstants.FILE_ID));
// if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) {
// String path = getFilePathInJxm(reportId, fileMetadata.getName());
// ((CSVDataSet) key).setProperty(ElementConstants.FILENAME, path);
// ((CSVDataSet) key).setProperty(ElementConstants.FILE_STORAGE, fileMetadata.getStorage());
// ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_NAME, fileMetadata.getName());
// ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_UPDATE_TIME, fileMetadata.getUpdateTime());
// ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_PROJECT_ID, fileMetadata.getProjectId());
// if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) {
// ((CSVDataSet) key).setProperty(ElementConstants.REF_FILE_ATTACH_INFO, fileMetadata.getAttachInfo());
// }
// }
// }
// }
// }
// private static void formatHttpFilePathForNode(Object key, String reportId, List<AttachmentBodyFile> fileList) {
// if (key == null) {
// return;
// }
// HTTPSamplerProxy source = (HTTPSamplerProxy) key;
// for (HTTPFileArg httpFileArg : source.getHTTPFiles()) {
// if (httpFileArg.getPropertyAsBoolean(ElementConstants.IS_REF)) {
// FileMetadataWithBLOBs fileMetadata = fileMetadataService.getFileMetadataById(
// httpFileArg.getPropertyAsString(ElementConstants.FILE_ID));
// if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) {
// String path = getFilePathInJxm(reportId, fileMetadata.getName());
// httpFileArg.setPath(path);
// httpFileArg.setProperty(ElementConstants.FILE_STORAGE, fileMetadata.getStorage());
// httpFileArg.setProperty(ElementConstants.REF_FILE_NAME, fileMetadata.getName());
// httpFileArg.setProperty(ElementConstants.REF_FILE_UPDATE_TIME, fileMetadata.getUpdateTime());
// httpFileArg.setProperty(ElementConstants.REF_FILE_PROJECT_ID, fileMetadata.getProjectId());
// if (StringUtils.isNotBlank(fileMetadata.getAttachInfo())) {
// httpFileArg.setProperty(ElementConstants.REF_FILE_ATTACH_INFO, fileMetadata.getAttachInfo());
// }
//
// fileList.add(new AttachmentBodyFile() {{
// this.setFileMetadataId(fileMetadata.getId());
// }});
// }
// }
// }
//
//
/**
* 获取当前jmx 涉及到的文件 执行时
*
@ -141,19 +322,12 @@ public class ApiFileUtil extends FileUtils {
if (source.getPropertyAsBoolean(ElementConstants.IS_REF)) {
FileMetadata fileMetadata = fileMetadataService.getFileMetadataById(
source.getPropertyAsString(ElementConstants.FILE_ID));
if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) {
file.setStorage(fileMetadata.getStorage());
file.setFileId(source.getPropertyAsString(ElementConstants.FILE_ID));
String fileName = StringUtils.join(reportId, File.separator, fileMetadata.getName());
file.setName(fileName);
String path = StringUtils.join(
BODY_FILE_DIR,
File.separator,
reportId,
File.separator,
fileMetadata.getName());
String path = getFilePathInJxm(reportId, fileMetadata.getName());
((CSVDataSet) key).setProperty(ElementConstants.FILENAME, path);
}
} else if (!new File(source.getPropertyAsString(ElementConstants.FILENAME)).exists()
@ -184,13 +358,7 @@ public class ApiFileUtil extends FileUtils {
file.setStorage(fileMetadata.getStorage());
file.setFileId(httpFileArg.getPropertyAsString(ElementConstants.FILE_ID));
file.setName(reportId + File.separator + fileMetadata.getName());
String path = StringUtils.join(
BODY_FILE_DIR,
File.separator,
reportId,
File.separator,
fileMetadata.getName());
String path = getFilePathInJxm(reportId, fileMetadata.getName());
httpFileArg.setPath(path);
}
} else if (!new File(httpFileArg.getPath()).exists()

2
api-test/backend/src/main/java/io/metersphere/commons/utils/HashTreeUtil.java
View File

@ -251,7 +251,7 @@ public class HashTreeUtil {
FileMetadataService fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class);
if (fileMetadataService != null) {
Map<String, byte[]> multipartFiles = new LinkedHashMap<>();
List<FileInfoDTO> fileInfoDTOList = fileMetadataService.downloadFileByIds(repositoryFileMap.keySet());
List<FileInfoDTO> fileInfoDTOList = fileMetadataService.downloadApiExecuteFilesByIds(repositoryFileMap.keySet());
fileInfoDTOList.forEach(dto -> {
if (ArrayUtils.isNotEmpty(dto.getFileByte())) {
String key = StringUtils.join(

1
api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java
View File

@ -66,6 +66,7 @@ public class ApiJMeterFileController {
.body(bytes);
}
@PostMapping("download/files")
public ResponseEntity<byte[]> downloadJmeterFiles(@RequestBody BodyFileRequest request) {
byte[] bytes = apiJmeterFileService.zipFilesToByteArray(request);

88
api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java
View File

@ -5,6 +5,7 @@ import io.metersphere.api.dto.EnvironmentType;
import io.metersphere.api.dto.definition.request.ElementUtil;
import io.metersphere.api.dto.definition.request.MsTestPlan;
import io.metersphere.api.exec.api.ApiCaseSerialService;
import io.metersphere.api.jmeter.utils.JmxFileUtil;
import io.metersphere.base.domain.*;
import io.metersphere.base.mapper.ApiExecutionQueueDetailMapper;
import io.metersphere.base.mapper.ApiScenarioMapper;
@ -13,6 +14,7 @@ import io.metersphere.base.mapper.plan.TestPlanApiScenarioMapper;
import io.metersphere.commons.constants.ApiRunMode;
import io.metersphere.commons.constants.PluginScenario;
import io.metersphere.commons.utils.*;
import io.metersphere.dto.AttachmentBodyFile;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.environment.service.BaseEnvGroupProjectService;
import io.metersphere.metadata.service.FileMetadataService;
@ -24,6 +26,7 @@ import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jorphan.collections.HashTree;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import java.io.ByteArrayOutputStream;
@ -50,6 +53,8 @@ public class ApiJMeterFileService {
private FileMetadataService fileMetadataService;
@Resource
private PluginMapper pluginMapper;
@Resource
private RedisTemplate<String, Object> redisTemplate;
// 接口测试 用例/接口
private static final List<String> CASE_MODES = new ArrayList<>() {{
@ -132,7 +137,7 @@ public class ApiJMeterFileService {
if (hashTree != null) {
ElementUtil.coverArguments(hashTree);
}
return zipFilesToByteArray((reportId + "_" + remoteTestId), hashTree);
return zipFilesToByteArray((reportId + "_" + remoteTestId), reportId, hashTree);
}
public byte[] downloadJmeterJar() {
@ -221,15 +226,6 @@ public class ApiJMeterFileService {
return jarFiles;
}
private Map<String, byte[]> getMultipartFiles(String reportId, HashTree hashTree) {
Map<String, byte[]> multipartFiles = new LinkedHashMap<>();
// 获取附件
List<BodyFile> files = new LinkedList<>();
ApiFileUtil.getExecuteFiles(hashTree, reportId, files);
HashTreeUtil.downFile(files, multipartFiles, fileMetadataService);
return multipartFiles;
}
private String replaceJmx(String jmx) {
jmx = StringUtils.replace(jmx, "<DubboSample", "<io.github.ningyu.jmeter.plugin.dubbo.sample.DubboSample");
jmx = StringUtils.replace(jmx, "</DubboSample>", "</io.github.ningyu.jmeter.plugin.dubbo.sample.DubboSample>");
@ -239,12 +235,25 @@ public class ApiJMeterFileService {
return jmx;
}
private byte[] zipFilesToByteArray(String testId, HashTree hashTree) {
String bodyFilePath = FileUtils.BODY_FILE_DIR;
private byte[] zipFilesToByteArray(String testId, String reportId, HashTree hashTree) {
String fileName = testId + ".jmx";
// 获取JMX使用到的附件
Map<String, byte[]> multipartFiles = this.getMultipartFiles(testId, hashTree);
/*
v2.7版本修改jmx附件逻辑：
在node执行机器设置临时文件目录。这里只提供jmx
node拿到jmx后解析jmx中包含的附件信息。附件通过以下流程来获取：
1 从node执行机的临时文件夹
2 临时文件夹中未找到的文件，根据文件类型来判断是否从minio/git下载，然后缓存到临时文件夹。
3 MinIO、Git中依然未能找到的文件（主要是Local文件），通过主工程下载，然后缓存到临时文件夹
所以这里不再使用以下方法来获取附件内容
Map<String, byte[]> multipartFiles = this.getMultipartFiles(testId, hashTree);
转为解析jmx中附件节点，赋予相关信息(例如文件关联类型、路径、更新时间等),并将文件信息存储在redis中，为了进行连接ms下载时的安全校验
*/
List<AttachmentBodyFile> attachmentBodyFileList = new ArrayList<>();
ApiFileUtil.formatFilePathForNode(hashTree, testId, attachmentBodyFileList);
if (CollectionUtils.isNotEmpty(attachmentBodyFileList)) {
redisTemplate.opsForValue().set(JmxFileUtil.REDIS_JMX_FILE_PREFIX + reportId, JmxFileUtil.getRedisJmxFileString(attachmentBodyFileList));
}
String jmx = new MsTestPlan().getJmx(hashTree);
// 处理dubbo请求生成jmx文件
@ -255,17 +264,6 @@ public class ApiJMeterFileService {
// 每个测试生成一个文件夹
files.put(fileName, jmx.getBytes(StandardCharsets.UTF_8));
if (multipartFiles != null && !multipartFiles.isEmpty()) {
for (String k : multipartFiles.keySet()) {
byte[] v = multipartFiles.get(k);
if (k.startsWith(bodyFilePath)) {
files.put(StringUtils.substringAfter(k, bodyFilePath), v);
} else {
LogUtil.error("WARNING:Attachment path is not in body_file_path: " + k);
files.put(k, v);
}
}
}
return listBytesToZip(files);
}
@ -290,10 +288,11 @@ public class ApiJMeterFileService {
public byte[] zipFilesToByteArray(BodyFileRequest request) {
Map<String, byte[]> files = new LinkedHashMap<>();
if (CollectionUtils.isNotEmpty(request.getBodyFiles())) {
List<BodyFile> bodyFiles = this.getLegalFiles(request);
LoggerUtil.info("开始从三方仓库下载文件");
HashTreeUtil.downFile(request.getBodyFiles(), files, fileMetadataService);
HashTreeUtil.downFile(bodyFiles, files, fileMetadataService);
LoggerUtil.info("从三方仓库下载文件");
for (BodyFile bodyFile : request.getBodyFiles()) {
for (BodyFile bodyFile : bodyFiles) {
File file = new File(bodyFile.getName());
if (!file.exists()) {
// 从MinIO下载
@ -308,7 +307,40 @@ public class ApiJMeterFileService {
}
}
}
return listBytesToZip(files);
Map<String, byte[]> zipFiles = new LinkedHashMap<>();
for (Map.Entry<String, byte[]> entry : files.entrySet()) {
//去除文件路径前的body路径
String filePath = entry.getKey();
if (StringUtils.startsWith(filePath, FileUtils.BODY_FILE_DIR + "/")) {
filePath = StringUtils.substring(filePath, FileUtils.BODY_FILE_DIR.length() + 1);
}
zipFiles.put(filePath, entry.getValue());
}
return listBytesToZip(zipFiles);
}
private List<BodyFile> getLegalFiles(BodyFileRequest request) {
List<BodyFile> returnList = new ArrayList<>();
Object jmxFileInfoObj = redisTemplate.opsForValue().get(JmxFileUtil.REDIS_JMX_FILE_PREFIX + request.getReportId());
List<AttachmentBodyFile> fileInJmx = JmxFileUtil.formatRedisJmxFileString(jmxFileInfoObj);
if (CollectionUtils.isNotEmpty(request.getBodyFiles())) {
request.getBodyFiles().forEach(attachmentBodyFile -> {
for (AttachmentBodyFile jmxFile : fileInJmx) {
if (StringUtils.isBlank(attachmentBodyFile.getRefResourceId())) {
if (StringUtils.equals(attachmentBodyFile.getName(), jmxFile.getFilePath())
&& StringUtils.equals(attachmentBodyFile.getName(), jmxFile.getName())) {
returnList.add(attachmentBodyFile);
}
} else {
if (StringUtils.equals(attachmentBodyFile.getRefResourceId(), jmxFile.getFileMetadataId())) {
returnList.add(attachmentBodyFile);
}
}
}
});
}
return returnList;
}
}

29
framework/sdk-parent/jmeter/src/main/java/io/metersphere/dto/AttachmentBodyFile.java Normal file
View File

@ -0,0 +1,29 @@
package io.metersphere.dto;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class AttachmentBodyFile {
private String id;
private String name;
// 调试附件处理
private String refResourceId;
//JMX中的路径
private String filePath;
//在文件服务器里的路径
private String remotePath;
//在本地节点的路径
private String localPath;
private boolean isRef;
private String fileMetadataId;
private String projectId;
private long FileUpdateTime;
private String fileStorage;
private String fileAttachInfoJson;
private byte[] fileBytes;
}

9
framework/sdk-parent/jmeter/src/main/java/io/metersphere/enums/JmxFileMetadataColumns.java Normal file
View File

@ -0,0 +1,9 @@
package io.metersphere.enums;
public enum JmxFileMetadataColumns {
REF_FILE_STORAGE,
REF_FILE_UPDATE_TIME,
REF_FILE_PROJECT_ID,
REF_FILE_ATTACH_INFO,
REF_FILE_NAME
}

130
framework/sdk-parent/jmeter/src/main/java/io/metersphere/utils/TemporaryFileUtil.java Normal file
View File

@ -0,0 +1,130 @@
package io.metersphere.utils;
import org.apache.commons.lang3.StringUtils;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
public class TemporaryFileUtil {
public final String fileFolder;
public static final String NODE_FILE_FOLDER = "node";
public static final String MS_FILE_FOLDER = "ms";
public static final String DEFAULT_FILE_FOLDER = "default";
public TemporaryFileUtil(String folder) {
if (StringUtils.isBlank(folder)) {
folder = DEFAULT_FILE_FOLDER;
}
this.fileFolder = File.separator + "opt"
+ File.separator + "metersphere"
+ File.separator + "data"
+ File.separator + "body"
+ File.separator + "local-file"
+ File.separator + folder
+ File.separator;
}
public String generateFileDir(String folder) {
if (StringUtils.isBlank(folder)) {
folder = DEFAULT_FILE_FOLDER;
}
return fileFolder + folder + File.separator;
}
public String generateFilePath(String folder, long updateTime, String fileName) {
String finalFileName = updateTime > 0 ? updateTime + "_" + fileName : fileName;
return generateFileDir(folder) + finalFileName;
}
public File getFile(String folder, long updateTime, String fileName) {
File file = new File(generateFilePath(folder, updateTime, fileName));
if (file.exists()) {
return file;
} else {
return null;
}
}
public void saveFile(String folder, long updateTime, String fileName, byte[] fileBytes) {
//删除过期文件
deleteOldFile(folder, fileName);
this.createFile(generateFilePath(folder, updateTime, fileName), fileBytes);
}
public void saveFileByParamCheck(String folder, long updateTime, String fileName, byte[] fileBytes) {
if (fileBytes != null && StringUtils.isNotBlank(folder) && updateTime > 0
&& StringUtils.isNotBlank(fileName) && fileBytes.length > 0) {
//删除过期文件
deleteOldFile(folder, fileName);
this.createFile(generateFilePath(folder, updateTime, fileName), fileBytes);
}
}
private void deleteOldFile(String folder, String deleteFileName) {
List<String> deleteFileList = new ArrayList<>();
File file = new File(generateFileDir(folder));
if (file.exists() && file.isDirectory()) {
String[] fileNameArr = file.list();
if (fileNameArr != null) {
for (String fileName : fileNameArr) {
if (fileName.endsWith("_" + deleteFileName)) {
deleteFileList.add(fileName);
}
}
}
}
deleteFileList.forEach(fileName -> this.deleteFile(generateFileDir(folder) + fileName));
}
public byte[] fileToByte(File tradeFile) {
byte[] buffer = null;
try (FileInputStream fis = new FileInputStream(tradeFile);
ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
byte[] b = new byte[1024];
int n;
while ((n = fis.read(b)) != -1) {
bos.write(b, 0, n);
}
buffer = bos.toByteArray();
} catch (Exception e) {
LoggerUtil.error(e);
}
return buffer;
}
private void createFile(String filePath, byte[] fileBytes) {
File file = new File(filePath);
if (file.exists()) {
this.deleteFile(filePath);
}
try {
File dir = file.getParentFile();
if (!dir.exists()) {
dir.mkdirs();
}
file.createNewFile();
} catch (Exception e) {
LoggerUtil.error(e);
}
try (InputStream in = new ByteArrayInputStream(fileBytes); OutputStream out = new FileOutputStream(file)) {
final int MAX = 4096;
byte[] buf = new byte[MAX];
for (int bytesRead = in.read(buf, 0, MAX); bytesRead != -1; bytesRead = in.read(buf, 0, MAX)) {
out.write(buf, 0, bytesRead);
}
} catch (IOException e) {
LoggerUtil.error(e);
}
}
public void deleteFile(String path) {
File file = new File(path);
if (file.exists()) {
file.delete();
}
}
}

2
framework/sdk-parent/sdk/src/main/java/io/metersphere/dto/FileInfoDTO.java
View File

@ -8,6 +8,8 @@ import lombok.Data;
public class FileInfoDTO {
private String id;
private String fileName;
private String projectId;
private long fileLastUpdateTime;
private String storage;
private String path;
private byte[] fileByte;

39
framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/GitFileRepository.java
View File

@ -35,14 +35,14 @@ public class GitFileRepository implements FileRepository {
@Override
public byte[] getFile(FileRequest request) throws Exception {
byte[] buffer = new byte[0];
byte[] fileBytes = new byte[0];
if (request.getFileAttachInfo() != null) {
RemoteFileAttachInfo gitFileInfo = request.getFileAttachInfo();
GitRepositoryUtil repositoryUtils = new GitRepositoryUtil(
gitFileInfo.getRepositoryPath(), gitFileInfo.getUserName(), gitFileInfo.getToken());
buffer = repositoryUtils.getSingleFile(gitFileInfo.getFilePath(), gitFileInfo.getCommitId());
fileBytes = repositoryUtils.getSingleFile(gitFileInfo.getFilePath(), gitFileInfo.getCommitId());
}
return buffer;
return fileBytes;
}
@Override
@ -71,32 +71,41 @@ public class GitFileRepository implements FileRepository {
List<FileRequest> requestList = entry.getValue();
RemoteFileAttachInfo baseGitFileInfo = null;
List<RepositoryRequest> repositoryRequestList = new ArrayList<>();
List<RepositoryRequest> downloadFileList = new ArrayList<>();
Map<String, byte[]> fileByteMap = new HashMap<>();
for (FileRequest fileRequest : requestList) {
RemoteFileAttachInfo gitFileInfo = fileRequest.getFileAttachInfo();
if (baseGitFileInfo == null) {
baseGitFileInfo = gitFileInfo;
}
repositoryRequestList.add(new RepositoryRequest() {{
downloadFileList.add(new RepositoryRequest() {{
this.setCommitId(gitFileInfo.getCommitId());
this.setFilePath(gitFileInfo.getFilePath());
this.setFileMetadataId(fileRequest.getResourceId());
this.setProjectId(fileRequest.getProjectId());
this.setUpdateTime(fileRequest.getUpdateTime());
}});
}
GitRepositoryUtil repositoryUtils = new GitRepositoryUtil(
baseGitFileInfo.getRepositoryPath(),
baseGitFileInfo.getUserName(), baseGitFileInfo.getToken());
if (CollectionUtils.isNotEmpty(downloadFileList) && baseGitFileInfo != null) {
GitRepositoryUtil repositoryUtils = new GitRepositoryUtil(
baseGitFileInfo.getRepositoryPath(),
baseGitFileInfo.getUserName(), baseGitFileInfo.getToken());
Map<String, byte[]> downloadFileMap = repositoryUtils.getFiles(downloadFileList);
fileByteMap.putAll(downloadFileMap);
}
Map<String, byte[]> fileByteMap = repositoryUtils.getFiles(repositoryRequestList);
repositoryRequestList.forEach(repositoryFile -> {
if (fileByteMap.get(repositoryFile.getFileMetadataId()) != null) {
requestList.forEach(fileRequest -> {
RemoteFileAttachInfo gitFileInfo = fileRequest.getFileAttachInfo();
if (fileByteMap.get(fileRequest.getResourceId()) != null) {
FileInfoDTO repositoryFileDTO = new FileInfoDTO(
repositoryFile.getFileMetadataId(),
MetadataUtils.getFileNameByRemotePath(repositoryFile.getFilePath()),
fileRequest.getResourceId(),
MetadataUtils.getFileNameByRemotePath(gitFileInfo.getFilePath()),
fileRequest.getProjectId(),
fileRequest.getUpdateTime(),
StorageConstants.GIT.name(),
repositoryFile.getFilePath(),
fileByteMap.get(repositoryFile.getFileMetadataId()));
gitFileInfo.getFilePath(),
fileByteMap.get(fileRequest.getResourceId()));
list.add(repositoryFileDTO);
}
});

1
framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/LocalFileRepository.java
View File

@ -119,6 +119,7 @@ public class LocalFileRepository implements FileRepository {
}
return new FileInfoDTO(
fileRequest.getResourceId(), fileRequest.getFileName(),
fileRequest.getProjectId(), fileRequest.getUpdateTime(),
fileRequest.getStorage(), fileRequest.getPath(), content);
}).collect(Collectors.toList());
}

4
framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/repository/MinIOFileRepository.java
View File

@ -7,12 +7,12 @@ import io.metersphere.dto.FileInfoDTO;
import io.metersphere.metadata.vo.FileRequest;
import io.minio.*;
import io.minio.messages.Item;
import jakarta.annotation.Resource;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import jakarta.annotation.Resource;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.ArrayList;
@ -133,7 +133,7 @@ public class MinIOFileRepository implements FileRepository {
List<FileInfoDTO> list = new ArrayList<>();
if (CollectionUtils.isNotEmpty(requestList)) {
for (FileRequest fileRequest : requestList) {
FileInfoDTO fileInfoDTO = new FileInfoDTO(fileRequest.getResourceId(), fileRequest.getFileName(), fileRequest.getStorage(), fileRequest.getPath(), this.getFile(fileRequest));
FileInfoDTO fileInfoDTO = new FileInfoDTO(fileRequest.getResourceId(), fileRequest.getFileName(), fileRequest.getProjectId(), fileRequest.getUpdateTime(), fileRequest.getStorage(), fileRequest.getPath(), this.getFile(fileRequest));
list.add(fileInfoDTO);
}
}

101
framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/service/FileMetadataService.java
View File

@ -10,10 +10,7 @@ import io.metersphere.commons.constants.ApiTestConstants;
import io.metersphere.commons.constants.FileModuleTypeConstants;
import io.metersphere.commons.constants.StorageConstants;
import io.metersphere.commons.exception.MSException;
import io.metersphere.commons.utils.FileUtils;
import io.metersphere.commons.utils.JSON;
import io.metersphere.commons.utils.LogUtil;
import io.metersphere.commons.utils.SessionUtils;
import io.metersphere.commons.utils.*;
import io.metersphere.dto.FileInfoDTO;
import io.metersphere.i18n.Translator;
import io.metersphere.log.utils.ReflexObjectUtil;
@ -25,6 +22,8 @@ import io.metersphere.metadata.utils.MetadataUtils;
import io.metersphere.metadata.vo.*;
import io.metersphere.request.OrderRequest;
import io.metersphere.request.QueryProjectFileRequest;
import io.metersphere.utils.TemporaryFileUtil;
import jakarta.annotation.Resource;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.http.HttpHeaders;
@ -33,7 +32,6 @@ import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import jakarta.annotation.Resource;
import java.io.File;
import java.io.InputStream;
import java.util.*;
@ -54,6 +52,8 @@ public class FileMetadataService {
@Resource
private FileAssociationMapper fileAssociationMapper;
private TemporaryFileUtil temporaryFileUtil;
public List<FileMetadata> create(FileMetadataCreateRequest fileMetadata, List<MultipartFile> files) {
List<FileMetadata> result = new ArrayList<>();
if (fileMetadata == null) {
@ -239,6 +239,7 @@ public class FileMetadataService {
request.setPath(fileMetadata.getPath());
request.setStorage(fileMetadata.getStorage());
request.setFileAttachInfoByString(fileMetadata.getAttachInfo());
request.setUpdateTime(fileMetadata.getUpdateTime());
bytes = fileManagerService.downloadFile(request);
}
return bytes;
@ -347,7 +348,7 @@ public class FileMetadataService {
return fileMetadata;
}
public FileMetadata getFileMetadataById(String fileId) {
public FileMetadataWithBLOBs getFileMetadataById(String fileId) {
return fileMetadataMapper.selectByPrimaryKey(fileId);
}
@ -520,8 +521,70 @@ public class FileMetadataService {
return fileMetadataList.stream().map(FileMetadata::getId).collect(Collectors.toList());
}
/**
* 接口测试执行时下载附件的方法。
* 该方法会优先判断是否存在已下载好的文件，避免多次执行造成多次下载的情况
*
* @param fileIdList
* @return
*/
public List<FileInfoDTO> downloadApiExecuteFilesByIds(Collection<String> fileIdList) {
if (temporaryFileUtil == null) {
temporaryFileUtil = CommonBeanFactory.getBean(TemporaryFileUtil.class);
}
List<FileInfoDTO> fileInfoDTOList = new ArrayList<>();
if (CollectionUtils.isEmpty(fileIdList)) {
return fileInfoDTOList;
}
LogUtil.info(JSON.toJSONString(fileIdList) + " 获取执行文件开始");
FileMetadataExample example = new FileMetadataExample();
example.createCriteria().andIdIn(new ArrayList<>(fileIdList));
List<FileMetadataWithBLOBs> fileMetadataWithBLOBList = fileMetadataMapper.selectByExampleWithBLOBs(example);
List<FileRequest> downloadFileRequest = new ArrayList<>();
//检查是否存在已下载的文件
fileMetadataWithBLOBList.forEach(fileMetadata -> {
File file = temporaryFileUtil.getFile(fileMetadata.getProjectId(), fileMetadata.getUpdateTime(), fileMetadata.getName());
if (file != null && file.exists() && file.isFile()) {
FileInfoDTO fileInfoDTO = new FileInfoDTO(fileMetadata.getId(), fileMetadata.getName(), fileMetadata.getProjectId(), fileMetadata.getUpdateTime(), fileMetadata.getStorage(), fileMetadata.getPath(), FileUtils.fileToByte(file));
fileInfoDTOList.add(fileInfoDTO);
} else {
downloadFileRequest.add(this.genFileRequest(fileMetadata));
}
});
List<FileInfoDTO> repositoryFileDTOList = fileManagerService.downloadFileBatch(downloadFileRequest);
//将文件存储到执行文件目录中，避免多次执行时触发多次下载
if (CollectionUtils.isNotEmpty(repositoryFileDTOList)) {
repositoryFileDTOList.forEach(repositoryFile -> temporaryFileUtil.saveFileByParamCheck(repositoryFile.getProjectId(), repositoryFile.getFileLastUpdateTime(), repositoryFile.getFileName(), repositoryFile.getFileByte()));
fileInfoDTOList.addAll(repositoryFileDTOList);
}
return fileInfoDTOList;
}
private FileRequest genFileRequest(FileMetadataWithBLOBs fileMetadata) {
if (fileMetadata != null) {
FileRequest request = new FileRequest(fileMetadata.getProjectId(), fileMetadata.getName(), fileMetadata.getType());
request.setResourceId(fileMetadata.getId());
request.setResourceType(fileMetadata.getResourceType());
request.setPath(fileMetadata.getPath());
request.setStorage(fileMetadata.getStorage());
request.setUpdateTime(fileMetadata.getUpdateTime());
if (StringUtils.equals(fileMetadata.getStorage(), StorageConstants.GIT.name())) {
try {
RemoteFileAttachInfo gitFileInfo = JSON.parseObject(fileMetadata.getAttachInfo(), RemoteFileAttachInfo.class);
request.setFileAttachInfo(gitFileInfo);
} catch (Exception e) {
LogUtil.error("解析Git附加信息【" + fileMetadata.getAttachInfo() + "】失败!", e);
}
}
return request;
} else {
return new FileRequest();
}
}
public List<FileInfoDTO> downloadFileByIds(Collection<String> fileIdList) {
if (org.apache.commons.collections.CollectionUtils.isEmpty(fileIdList)) {
if (CollectionUtils.isEmpty(fileIdList)) {
return new ArrayList<>(0);
}
LogUtil.info(JSON.toJSONString(fileIdList) + " 获取文件开始");
@ -532,22 +595,8 @@ public class FileMetadataService {
List<FileRequest> requestList = new ArrayList<>();
fileMetadataWithBLOBs.forEach(fileMetadata -> {
FileRequest request = new FileRequest(fileMetadata.getProjectId(), fileMetadata.getName(), fileMetadata.getType());
request.setResourceId(fileMetadata.getId());
request.setResourceType(fileMetadata.getResourceType());
request.setPath(fileMetadata.getPath());
request.setStorage(fileMetadata.getStorage());
if (StringUtils.equals(fileMetadata.getStorage(), StorageConstants.GIT.name())) {
try {
RemoteFileAttachInfo gitFileInfo = JSON.parseObject(fileMetadata.getAttachInfo(), RemoteFileAttachInfo.class);
request.setFileAttachInfo(gitFileInfo);
} catch (Exception e) {
LogUtil.error("解析Git附加信息【" + fileMetadata.getAttachInfo() + "】失败!", e);
}
}
requestList.add(request);
requestList.add(this.genFileRequest(fileMetadata));
});
List<FileInfoDTO> repositoryFileDTOList = fileManagerService.downloadFileBatch(requestList);
LogUtil.info(JSON.toJSONString(fileIdList) + " 获取文件结束。");
return repositoryFileDTOList;
@ -564,12 +613,12 @@ public class FileMetadataService {
RemoteFileAttachInfo gitFileAttachInfo = repositoryUtils.selectLastCommitIdByBranch(baseAttachInfo.getBranch(), baseAttachInfo.getFilePath());
if (gitFileAttachInfo != null &&
!StringUtils.equals(gitFileAttachInfo.getCommitId(), baseAttachInfo.getCommitId())) {
//有新的commitId，更新filemetadata的版本
long thistime = System.currentTimeMillis();
FileMetadataWithBLOBs newMetadata = this.genOtherVersionFileMetadata(baseMetadata, thistime, gitFileAttachInfo);
//有新的commitId，更新fileMetadata的版本
long thisTime = System.currentTimeMillis();
FileMetadataWithBLOBs newMetadata = this.genOtherVersionFileMetadata(baseMetadata, thisTime, gitFileAttachInfo);
fileMetadataMapper.insert(newMetadata);
baseMetadata.setUpdateTime(thistime);
baseMetadata.setUpdateTime(thisTime);
baseMetadata.setLatest(false);
baseMetadata.setUpdateUser(SessionUtils.getUserId());
fileMetadataMapper.updateByPrimaryKeySelective(baseMetadata);

6
framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/FileRequest.java
View File

@ -21,6 +21,10 @@ public class FileRequest {
//文件附属信息
private RemoteFileAttachInfo fileAttachInfo;
//文件更新时间。 用于在缓存文件中判断是否需要更新
private long updateTime;
//缓存文件夹中通过moduleId作为内部文件结构
public FileRequest() {
}
@ -30,7 +34,7 @@ public class FileRequest {
this.projectId = projectId;
this.type = StringUtils.isNotEmpty(type) ? type.toLowerCase() : null;
this.fileName = name;
if (StringUtils.isNotEmpty(this.type) && !name.endsWith(this.type)) {
if (!StringUtils.equalsIgnoreCase(this.storage, StorageConstants.MINIO.name()) && StringUtils.isNotEmpty(this.type) && !name.endsWith(this.type)) {
this.fileName = StringUtils.join(name, ".", this.type);
}
}

2
framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/vo/RepositoryRequest.java
View File

@ -7,4 +7,6 @@ public class RepositoryRequest {
private String fileMetadataId;
private String filePath;
private String commitId;
private String projectId;
private long updateTime;
}

68
project-management/frontend/src/business/menu/file/list/FileVersionList.vue
View File

@ -1,19 +1,35 @@
<template>
<div>
<el-table :data="data" class="test-content document-table" style="width: 100%" ref="table">
<el-table-column prop="commitId"
:label="$t('project.project_file.repository.file_version')"
min-width="120px"
show-overflow-tooltip/>
<el-table-column prop="commitMessage"
:label="$t('project.project_file.repository.update_log')"
min-width="200"
show-overflow-tooltip/>
<el-table-column prop="operator"
:label="$t('operating_log.user')"
min-width="120"
show-overflow-tooltip/>
<el-table-column prop="operatorTime" min-width="160" :label="$t('operating_log.time')" sortable>
<el-table
:data="data"
class="test-content document-table"
style="width: 100%"
ref="table"
>
<el-table-column
prop="commitId"
:label="$t('project.project_file.repository.file_version')"
min-width="120px"
show-overflow-tooltip
/>
<el-table-column
prop="commitMessage"
:label="$t('project.project_file.repository.update_log')"
min-width="200"
show-overflow-tooltip
/>
<el-table-column
prop="operator"
:label="$t('operating_log.user')"
min-width="120"
show-overflow-tooltip
/>
<el-table-column
prop="operatorTime"
min-width="160"
:label="$t('operating_log.time')"
sortable
>
<template v-slot:default="scope">
<span>{{ scope.row.operatorTime | datetimeFormat }}</span>
</template>
@ -23,7 +39,7 @@
</template>
<script>
import {getFileVersion} from "@/api/file";
import { getFileVersion } from "@/api/file";
export default {
name: "FileVersionList",
@ -38,7 +54,7 @@ export default {
watch: {
fileMetadataRefId() {
this.selectData();
}
},
},
created() {
this.selectData();
@ -46,21 +62,21 @@ export default {
methods: {
selectData() {
if (this.fileMetadataRefId) {
getFileVersion(this.fileMetadataRefId).then(response => {
getFileVersion(this.fileMetadataRefId).then((response) => {
if (response.data) {
this.data = response.data;
}
this.$nextTick(() => {
this.$refs.table.doLayout();
})
if (this.$refs.table) {
//加这一层判断时是因为有可能出现还未等到接口出现返回就关闭了窗口，导致前台报错
this.$refs.table.doLayout();
}
});
});
}
}
}
}
},
},
};
</script>
<style scoped>
</style>
<style scoped></style>