test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(接口定义): 优化服务下载jar包 

--story=1011108 --user=王孝刚 资源池加载插件直接从MinIO获取
https://www.tapd.cn/55049933/s/1331634
This commit is contained in:
wxg0103 2023-02-09 17:48:04 +08:00 committed by fit2-zhao
parent 29e22670d1
commit 5666dea7de
19 changed files with 311 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api-test/backend/src/main/java/io/metersphere/api/dto/definition/request/MsTestPlan.java
View File

@ -23,8 +23,8 @@ import java.util.List;
public class MsTestPlan extends MsTestElement {
private String type = ElementConstants.TEST_PLAN;
private String clazzName = MsTestPlan.class.getCanonicalName();
// 自定义JAR
private List<String> jarPaths;
// 自定义JAR项目id
private List<String> projectJarIds;
private boolean serializeThreadGroups = false;
@ -47,8 +47,8 @@ public class MsTestPlan extends MsTestElement {
testPlan.setFunctionalMode(false);
testPlan.setSerialized(serializeThreadGroups);
testPlan.setTearDownOnShutdown(true);
if (CollectionUtils.isNotEmpty(jarPaths)) {
testPlan.setProperty(ApiTestConstants.JAR_PATH, JSON.toJSONString(jarPaths));
if (CollectionUtils.isNotEmpty(projectJarIds)) {
testPlan.setProperty(ApiTestConstants.JAR_PATH, JSON.toJSONString(projectJarIds));
}
testPlan.setUserDefinedVariables(new Arguments());
return testPlan;

8
api-test/backend/src/main/java/io/metersphere/api/exec/api/ApiCaseSerialService.java
View File

@ -32,9 +32,10 @@ import io.metersphere.dto.ResultDTO;
import io.metersphere.environment.service.BaseEnvironmentService;
import io.metersphere.plugin.core.MsTestElement;
import io.metersphere.service.ApiExecutionQueueService;
import io.metersphere.service.ApiRetryOnFailureService;
import io.metersphere.service.RemakeReportService;
import io.metersphere.utils.LoggerUtil;
import io.metersphere.service.ApiRetryOnFailureService;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.StringUtils;
import org.apache.jorphan.collections.HashTree;
import org.json.JSONObject;
@ -42,7 +43,6 @@ import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import jakarta.annotation.Resource;
import java.util.*;
@Service
@ -135,9 +135,9 @@ public class ApiCaseSerialService {
if (!runRequest.getPool().isPool()) {
// 获取自定义JAR
String projectId = caseWithBLOBs.getProjectId();
testPlan.setJarPaths(NewDriverManager.getJars(new ArrayList<>() {{
testPlan.setProjectJarIds(NewDriverManager.getJars(new ArrayList<>() {{
this.add(projectId);
}}));
}}, runRequest.getPool()).keySet().stream().toList());
}
testPlan.setHashTree(new LinkedList<>());
MsThreadGroup group = new MsThreadGroup();

41
api-test/backend/src/main/java/io/metersphere/api/exec/api/ApiExecuteService.java
View File

@ -20,25 +20,21 @@ import io.metersphere.base.mapper.ApiDefinitionMapper;
import io.metersphere.base.mapper.ApiTestCaseMapper;
import io.metersphere.base.mapper.ext.ExtApiTestCaseMapper;
import io.metersphere.base.mapper.plan.TestPlanApiCaseMapper;
import io.metersphere.commons.constants.ApiRunMode;
import io.metersphere.commons.constants.CommonConstants;
import io.metersphere.commons.constants.ElementConstants;
import io.metersphere.commons.constants.ExtendedParameter;
import io.metersphere.commons.constants.*;
import io.metersphere.commons.enums.ApiReportStatus;
import io.metersphere.commons.utils.*;
import io.metersphere.dto.BaseSystemConfigDTO;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.dto.MsExecResponseDTO;
import io.metersphere.dto.RunModeConfigDTO;
import io.metersphere.dto.*;
import io.metersphere.environment.service.BaseEnvironmentService;
import io.metersphere.plugin.core.MsTestElement;
import io.metersphere.service.SystemParameterService;
import io.metersphere.service.definition.TcpApiParamService;
import io.metersphere.utils.LoggerUtil;
import io.metersphere.vo.BooleanPool;
import jakarta.annotation.Resource;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jmeter.testelement.TestPlan;
import org.apache.jorphan.collections.HashTree;
import org.apache.jorphan.collections.ListedHashTree;
import org.json.JSONObject;
@ -46,7 +42,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import jakarta.annotation.Resource;
import java.util.*;
@Service
@ -85,7 +80,8 @@ public class ApiExecuteService {
if (caseWithBLOBs == null) {
return null;
}
jMeterService.verifyPool(caseWithBLOBs.getProjectId(), new RunModeConfigDTO());
RunModeConfigDTO runModeConfigDTO = new RunModeConfigDTO();
jMeterService.verifyPool(caseWithBLOBs.getProjectId(), runModeConfigDTO);
if (StringUtils.isBlank(request.getEnvironmentId())) {
request.setEnvironmentId(extApiTestCaseMapper.getApiCaseEnvironment(request.getCaseId()));
@ -132,7 +128,7 @@ public class ApiExecuteService {
// 多态JSON普通转换会丢失内容，需要通过 ObjectMapper 获取
if (testCaseWithBLOBs != null && StringUtils.isNotEmpty(testCaseWithBLOBs.getRequest())) {
try {
HashTree jmeterHashTree = this.generateHashTree(request, testCaseWithBLOBs);
HashTree jmeterHashTree = this.generateHashTree(request, testCaseWithBLOBs, runModeConfigDTO);
if (LoggerUtil.getLogger().isDebugEnabled()) {
LoggerUtil.debug("生成jmx文件：" + ElementUtil.hashTreeToString(jmeterHashTree));
}
@ -192,9 +188,9 @@ public class ApiExecuteService {
threadGroup.setName(MsThreadGroup.class.getCanonicalName());
threadGroup.setHashTree(new LinkedList<>());
testPlan.getHashTree().add(threadGroup);
testPlan.setJarPaths(NewDriverManager.getJars(new ArrayList<>() {{
testPlan.setProjectJarIds(NewDriverManager.getJars(new ArrayList<>() {{
this.add(request.getProjectId());
}}));
}}, new BooleanPool()).keySet().stream().toList());
threadGroup.getHashTree().add(request);
ParameterConfig config = new ParameterConfig();
config.setProjectId(request.getProjectId());
@ -244,8 +240,6 @@ public class ApiExecuteService {
runMode = ApiRunMode.API_PLAN.name();
}
// 加载自定义JAR
List<String> projectIds = NewDriverManager.loadJar(request);
HashTree hashTree = request.getTestElement().generateHashTree(config);
String jmx = request.getTestElement().getJmx(hashTree);
LoggerUtil.info("生成执行JMX内容【 " + jmx + " 】");
@ -260,9 +254,6 @@ public class ApiExecuteService {
this.put(CommonConstants.USER_ID, SessionUtils.getUser().getId());
this.put("userName", SessionUtils.getUser().getName());
}});
if (CollectionUtils.isNotEmpty(projectIds)) {
runRequest.getExtendedParameters().put(ExtendedParameter.PROJECT_ID, JSON.toJSONString(projectIds));
}
// 开始执行
if (StringUtils.isNotEmpty(request.getConfig().getResourcePoolId())) {
runRequest.setPool(GenerateHashTreeUtil.isResourcePool(request.getConfig().getResourcePoolId()));
@ -270,10 +261,17 @@ public class ApiExecuteService {
BaseSystemConfigDTO baseInfo = systemParameterService.getBaseInfo();
runRequest.setPlatformUrl(GenerateHashTreeUtil.getPlatformUrl(baseInfo, runRequest, null));
}
// 加载自定义JAR
Map<String, List<ProjectJarConfig>> projectJarMap = NewDriverManager.loadJar(request, runRequest.getPool());
if (MapUtils.isNotEmpty(projectJarMap)) {
TestPlan test = (TestPlan) runRequest.getHashTree().getArray()[0];
test.setProperty(ApiTestConstants.JAR_PATH, JSON.toJSONString(projectJarMap.keySet().stream().toList()));
runRequest.getExtendedParameters().put(ExtendedParameter.PROJECT_JAR_MAP, JSON.toJSONString(projectJarMap));
}
return runRequest;
}
public HashTree generateHashTree(RunCaseRequest request, ApiTestCaseWithBLOBs testCaseWithBLOBs) throws Exception {
public HashTree generateHashTree(RunCaseRequest request, ApiTestCaseWithBLOBs testCaseWithBLOBs, RunModeConfigDTO runModeConfigDTO) throws Exception {
PerformInspectionUtil.countMatches(testCaseWithBLOBs.getRequest(), testCaseWithBLOBs.getId());
JSONObject elementObj = JSONUtil.parseObject(testCaseWithBLOBs.getRequest());
ElementUtil.dataFormatting(elementObj);
@ -295,9 +293,10 @@ public class ApiExecuteService {
MsTestPlan testPlan = new MsTestPlan();
// 获取自定义JAR
String projectId = testCaseWithBLOBs.getProjectId();
testPlan.setJarPaths(NewDriverManager.getJars(new ArrayList<>() {{
BooleanPool pool = GenerateHashTreeUtil.isResourcePool(runModeConfigDTO.getResourcePoolId());
testPlan.setProjectJarIds(NewDriverManager.getJars(new ArrayList<>() {{
this.add(projectId);
}}));
}}, pool).keySet().stream().toList());
testPlan.setHashTree(new LinkedList<>());
HashTree jmeterHashTree = new ListedHashTree();

19
api-test/backend/src/main/java/io/metersphere/api/exec/scenario/ApiScenarioExecuteService.java
View File

@ -22,6 +22,7 @@ import io.metersphere.base.mapper.ApiScenarioReportMapper;
import io.metersphere.base.mapper.ext.ExtApiScenarioMapper;
import io.metersphere.base.mapper.plan.ext.ExtTestPlanScenarioCaseMapper;
import io.metersphere.commons.constants.ApiRunMode;
import io.metersphere.commons.constants.ApiTestConstants;
import io.metersphere.commons.constants.ExtendedParameter;
import io.metersphere.commons.constants.ReportTriggerMode;
import io.metersphere.commons.enums.ApiReportStatus;
@ -29,10 +30,7 @@ import io.metersphere.commons.exception.MSException;
import io.metersphere.commons.utils.*;
import io.metersphere.commons.vo.RunPlanScenarioVO;
import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.BaseSystemConfigDTO;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.dto.MsExecResponseDTO;
import io.metersphere.dto.RunModeConfigDTO;
import io.metersphere.dto.*;
import io.metersphere.environment.service.BaseEnvGroupProjectService;
import io.metersphere.i18n.Translator;
import io.metersphere.plugin.core.MsTestElement;
@ -43,18 +41,19 @@ import io.metersphere.service.definition.TcpApiParamService;
import io.metersphere.service.scenario.ApiScenarioReportService;
import io.metersphere.service.scenario.ApiScenarioReportStructureService;
import io.metersphere.utils.LoggerUtil;
import jakarta.annotation.Resource;
import org.apache.commons.beanutils.BeanComparator;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.collections4.comparators.FixedOrderComparator;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jmeter.testelement.TestPlan;
import org.apache.jorphan.collections.HashTree;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import jakarta.annotation.Resource;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -437,8 +436,6 @@ public class ApiScenarioExecuteService {
uploadBodyFiles(request.getBodyFileRequestIds(), bodyFiles);
FileUtils.createBodyFiles(request.getScenarioFileIds(), scenarioFiles);
this.testElement(request);
// 加载自定义JAR
List<String> projectIds = NewDriverManager.loadJar(request);
HashTree hashTree = request.getTestElement().generateHashTree(config);
String runMode = StringUtils.isEmpty(request.getRunMode()) ? ApiRunMode.SCENARIO.name() : request.getRunMode();
JmeterRunRequestDTO runRequest = new JmeterRunRequestDTO(request.getId(), request.getId(), runMode, hashTree);
@ -453,8 +450,12 @@ public class ApiScenarioExecuteService {
BaseSystemConfigDTO baseInfo = systemParameterService.getBaseInfo();
runRequest.setPlatformUrl(GenerateHashTreeUtil.getPlatformUrl(baseInfo, runRequest, null));
}
if (CollectionUtils.isNotEmpty(projectIds)) {
runRequest.getExtendedParameters().put(ExtendedParameter.PROJECT_ID, JSON.toJSONString(projectIds));
// 加载自定义JAR
Map<String, List<ProjectJarConfig>> loadJar = NewDriverManager.loadJar(request, runRequest.getPool());
if (MapUtils.isNotEmpty(loadJar)) {
TestPlan test = (TestPlan) runRequest.getHashTree().getArray()[0];
test.setProperty(ApiTestConstants.JAR_PATH, JSON.toJSONString(loadJar.keySet().stream().toList()));
runRequest.getExtendedParameters().put(ExtendedParameter.PROJECT_JAR_MAP, JSON.toJSONString(loadJar));
}
jMeterService.run(runRequest);
return request.getId();

75
api-test/backend/src/main/java/io/metersphere/api/jmeter/NewDriverManager.java
View File

@ -4,34 +4,84 @@ import io.metersphere.api.dto.definition.RunDefinitionRequest;
import io.metersphere.api.dto.definition.request.MsTestPlan;
import io.metersphere.base.domain.FileMetadata;
import io.metersphere.base.domain.FileMetadataExample;
import io.metersphere.base.domain.FileMetadataWithBLOBs;
import io.metersphere.base.mapper.FileMetadataMapper;
import io.metersphere.commons.constants.StorageConstants;
import io.metersphere.commons.utils.ApiFileUtil;
import io.metersphere.commons.utils.CommonBeanFactory;
import io.metersphere.dto.FileInfoDTO;
import io.metersphere.dto.ProjectJarConfig;
import io.metersphere.metadata.service.FileMetadataService;
import io.metersphere.utils.JarConfigUtils;
import io.metersphere.vo.BooleanPool;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class NewDriverManager {
public static List<String> getJars(List<String> projectIds) {
public static Map<String, List<ProjectJarConfig>> getJars(List<String> projectIds, BooleanPool pool) {
FileMetadataExample fileMetadata = new FileMetadataExample();
fileMetadata.createCriteria().andProjectIdIn(projectIds).andLoadJarEqualTo(true);
FileMetadataMapper fileMetadataMapper = CommonBeanFactory.getBean(FileMetadataMapper.class);
List<FileMetadata> files = fileMetadataMapper.selectByExample(fileMetadata);
files = files.stream().filter(s -> StringUtils.isNotEmpty(s.getPath())).collect(Collectors.toList());
// 获取文件内容
FileMetadataService fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class);
List<FileInfoDTO> fileInfoDTOS = fileMetadataService.downloadFileByIds(files.stream().map(FileMetadata::getId).collect(Collectors.toList()));
ApiFileUtil.createFiles(fileInfoDTOS);
return files.stream().map(FileMetadata::getPath).collect(Collectors.toList());
List<FileMetadataWithBLOBs> files = fileMetadataMapper.selectByExampleWithBLOBs(fileMetadata);
Map<String, List<ProjectJarConfig>> jarConfigMap = files.stream()
.collect(Collectors.groupingBy(FileMetadata::getProjectId, Collectors.mapping(
item -> {
ProjectJarConfig configs = new ProjectJarConfig();
configs.setId(item.getId());
configs.setName(item.getName());
configs.setStorage(item.getStorage());
//历史数据(存在数据库)
if (StringUtils.isEmpty(item.getStorage()) && StringUtils.isEmpty(item.getResourceType())) {
configs.setHasFile(true);
} else {
configs.setHasFile(false);
}
configs.setUpdateTime(item.getUpdateTime());
if (StringUtils.isNotEmpty(item.getStorage()) && StringUtils.equals(item.getStorage(), StorageConstants.GIT.name())) {
configs.setAttachInfo(item.getAttachInfo());
}
return configs;
}, Collectors.toList())));
if (!pool.isPool() && !pool.isK8s()) {
//获取本地
//获取需要下载的jar的map
Map<String, List<ProjectJarConfig>> map = JarConfigUtils.getJarConfigs(projectIds, jarConfigMap);
if (MapUtils.isNotEmpty(map)) {
//获取文件内容
FileMetadataService fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class);
map.forEach((key, value) -> {
//历史数据
value.stream().filter(s -> s.isHasFile()).forEach(s -> {
//获取文件内容
byte[] bytes = new byte[0];
// 兼容历史数据
bytes = fileMetadataService.getContent(s.getId());
ApiFileUtil.createFile(StringUtils.join(ApiFileUtil.LOCAL_JAR,
File.separator,
key,
File.separator,
s.getId(),
File.separator,
String.valueOf(s.getUpdateTime()), ".jar"), bytes);
});
List<String> jarIds = value.stream().filter(s -> !s.isHasFile()).map(ProjectJarConfig::getId).collect(Collectors.toList());
List<FileInfoDTO> fileInfoDTOS = fileMetadataService.downloadFileByIds(jarIds);
ApiFileUtil.createFiles(fileInfoDTOS, key, value);
});
}
}
return jarConfigMap;
}
public static List<String> loadJar(RunDefinitionRequest request) {
public static Map<String, List<ProjectJarConfig>> loadJar(RunDefinitionRequest request, BooleanPool pool) {
// 加载自定义JAR
MsTestPlan testPlan = (MsTestPlan) request.getTestElement();
List<String> projectIds = new ArrayList<>();
@ -43,7 +93,10 @@ public class NewDriverManager {
}
});
}
testPlan.setJarPaths(getJars(projectIds));
return projectIds;
Map<String, List<ProjectJarConfig>> jars = getJars(projectIds, pool);
testPlan.setProjectJarIds(projectIds);
return jars;
}
}

7
api-test/backend/src/main/java/io/metersphere/commons/config/UtilsConfig.java
View File

@ -1,5 +1,7 @@
package io.metersphere.commons.config;
import io.metersphere.commons.utils.FileUtils;
import io.metersphere.utils.LocalPathUtil;
import io.metersphere.utils.TemporaryFileUtil;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ -10,4 +12,9 @@ public class UtilsConfig {
public TemporaryFileUtil temporaryFileUtil() {
return new TemporaryFileUtil(TemporaryFileUtil.MS_FILE_FOLDER);
}
@Bean
public void localPathUtil() {
LocalPathUtil.prePath = FileUtils.LOCAL_JAR;
}
}

1
api-test/backend/src/main/java/io/metersphere/commons/constants/ExtendedParameter.java
View File

@ -5,4 +5,5 @@ public class ExtendedParameter {
public static final String SAVE_RESULT = "SAVE_RESULT";
public static final String PROJECT_ID = "projectId";
public static final String TEST_END = "TEST_END";
public static final String PROJECT_JAR_MAP = "PROJECT_JAR_MAP";
}

108
api-test/backend/src/main/java/io/metersphere/commons/utils/ApiFileUtil.java
View File

@ -1,16 +1,15 @@
package io.metersphere.commons.utils;
import io.metersphere.base.domain.FileMetadata;
import io.metersphere.base.domain.FileMetadataWithBLOBs;
import io.metersphere.commons.constants.ElementConstants;
import io.metersphere.commons.constants.StorageConstants;
import io.metersphere.dto.AttachmentBodyFile;
import io.metersphere.dto.FileInfoDTO;
import io.metersphere.dto.ProjectJarConfig;
import io.metersphere.enums.JmxFileMetadataColumns;
import io.metersphere.metadata.service.FileManagerService;
import io.metersphere.metadata.service.FileMetadataService;
import io.metersphere.metadata.vo.FileRequest;
import io.metersphere.request.BodyFile;
import io.metersphere.utils.LoggerUtil;
import io.metersphere.utils.TemporaryFileUtil;
import org.apache.commons.collections.CollectionUtils;
@ -32,18 +31,19 @@ public class ApiFileUtil extends FileUtils {
private static FileMetadataService fileMetadataService;
private static TemporaryFileUtil temporaryFileUtil;
public static String getFilePath(BodyFile file) {
String type = StringUtils.isNotEmpty(file.getFileType()) ? file.getFileType().toLowerCase() : null;
String name = file.getName();
if (type != null && !name.endsWith(type)) {
name = StringUtils.join(name, ".", type);
}
return StringUtils.join(ApiFileUtil.BODY_FILE_DIR, File.separator, file.getProjectId(), File.separator, name);
}
public static void createFiles(List<FileInfoDTO> infoDTOS) {
public static void createFiles(List<FileInfoDTO> infoDTOS, String key, List<ProjectJarConfig> value) {
infoDTOS.forEach(item -> {
createFile(item.getPath(), item.getFileByte());
value.forEach(config -> {
if (StringUtils.equals(item.getId(), config.getId())) {
createFile(StringUtils.join(ApiFileUtil.LOCAL_JAR,
File.separator,
key,
File.separator,
config.getId(),
File.separator,
String.valueOf(config.getUpdateTime()), ".jar"), item.getFileByte());
}
});
});
}
@ -233,86 +233,4 @@ public class ApiFileUtil extends FileUtils {
return StringUtils.join(BODY_FILE_DIR, File.separator, reportId, File.separator, fileName);
}
/**
* 获取当前jmx 涉及到的文件 执行时
*
* @param tree
*/
public static void getExecuteFiles(HashTree tree, String reportId, List<BodyFile> files) {
if (fileMetadataService == null) {
fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class);
}
for (Object key : tree.keySet()) {
if (key == null) {
continue;
}
HashTree node = tree.get(key);
if (key instanceof HTTPSamplerProxy) {
dealWithHttp(key, reportId, files);
} else if (key instanceof CSVDataSet) {
dealWithCsv(key, reportId, files);
}
if (node != null) {
getExecuteFiles(node, reportId, files);
}
}
}
private static void dealWithCsv(Object key, String reportId, List<BodyFile> files) {
CSVDataSet source = (CSVDataSet) key;
if (StringUtils.isNotEmpty(source.getPropertyAsString(ElementConstants.FILENAME))) {
BodyFile file = new BodyFile();
file.setId(source.getPropertyAsString(ElementConstants.FILENAME));
file.setName(source.getPropertyAsString(ElementConstants.FILENAME));
if (source.getPropertyAsBoolean(ElementConstants.IS_REF)) {
FileMetadata fileMetadata = fileMetadataService.getFileMetadataById(
source.getPropertyAsString(ElementConstants.FILE_ID));
if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) {
file.setStorage(fileMetadata.getStorage());
file.setFileId(source.getPropertyAsString(ElementConstants.FILE_ID));
String fileName = StringUtils.join(reportId, File.separator, fileMetadata.getName());
file.setName(fileName);
String path = getFilePathInJxm(reportId, fileMetadata.getName());
((CSVDataSet) key).setProperty(ElementConstants.FILENAME, path);
}
} else if (!new File(source.getPropertyAsString(ElementConstants.FILENAME)).exists()
&& StringUtils.isNotBlank(source.getPropertyAsString(ElementConstants.RESOURCE_ID))) {
// 从MinIO下载
downloadFile(source.getPropertyAsString(ElementConstants.RESOURCE_ID),
source.getPropertyAsString(ElementConstants.FILENAME));
}
files.add(file);
}
}
private static void dealWithHttp(Object key, String reportId, List<BodyFile> files) {
HTTPSamplerProxy source = (HTTPSamplerProxy) key;
if (source == null || source.getHTTPFiles().length == 0) {
return;
}
for (HTTPFileArg httpFileArg : source.getHTTPFiles()) {
BodyFile file = new BodyFile();
file.setId(httpFileArg.getParamName());
file.setName(httpFileArg.getPath());
if (httpFileArg.getPropertyAsBoolean(ElementConstants.IS_REF)) {
FileMetadata fileMetadata = fileMetadataService.getFileMetadataById(
httpFileArg.getPropertyAsString(ElementConstants.FILE_ID));
if (fileMetadata != null && !StringUtils.equals(fileMetadata.getStorage(), StorageConstants.LOCAL.name())) {
file.setStorage(fileMetadata.getStorage());
file.setFileId(httpFileArg.getPropertyAsString(ElementConstants.FILE_ID));
file.setName(reportId + File.separator + fileMetadata.getName());
String path = getFilePathInJxm(reportId, fileMetadata.getName());
httpFileArg.setPath(path);
}
} else if (!new File(httpFileArg.getPath()).exists()
&& StringUtils.isNotBlank(httpFileArg.getPropertyAsString(ElementConstants.RESOURCE_ID))) {
// 从MinIO下载
downloadFile(httpFileArg.getPropertyAsString(ElementConstants.RESOURCE_ID), httpFileArg.getPath());
}
files.add(file);
}
}
}

4
api-test/backend/src/main/java/io/metersphere/commons/utils/GenerateHashTreeUtil.java
View File

@ -20,10 +20,10 @@ import io.metersphere.dto.RunModeConfigDTO;
import io.metersphere.environment.service.BaseEnvGroupProjectService;
import io.metersphere.plugin.core.MsTestElement;
import io.metersphere.service.ApiExecutionQueueService;
import io.metersphere.service.ApiRetryOnFailureService;
import io.metersphere.service.RemakeReportService;
import io.metersphere.utils.LoggerUtil;
import io.metersphere.vo.BooleanPool;
import io.metersphere.service.ApiRetryOnFailureService;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jorphan.collections.HashTree;
@ -122,7 +122,7 @@ public class GenerateHashTreeUtil {
projectIds.add(k);
});
}
testPlan.setJarPaths(NewDriverManager.getJars(projectIds));
testPlan.setProjectJarIds(NewDriverManager.getJars(projectIds, runRequest.getPool()).keySet().stream().toList());
}
testPlan.setHashTree(new LinkedList<>());
try {

8
api-test/backend/src/main/java/io/metersphere/controller/ApiJMeterFileController.java
View File

@ -3,6 +3,7 @@ package io.metersphere.controller;
import io.metersphere.api.dto.BodyFileRequest;
import io.metersphere.api.jmeter.JMeterThreadUtils;
import io.metersphere.api.jmeter.utils.JmxFileUtil;
import io.metersphere.dto.ProjectJarConfig;
import io.metersphere.service.ApiJMeterFileService;
import io.metersphere.utils.LoggerUtil;
import jakarta.annotation.Resource;
@ -13,6 +14,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@RestController
@ -38,9 +40,9 @@ public class ApiJMeterFileController {
.body(bytes);
}
@GetMapping("download/jar/{projectId}")
public ResponseEntity<byte[]> downloadJmeterFiles(@PathVariable String projectId) {
byte[] bytes = apiJmeterFileService.downloadJmeterJar(projectId);
@PostMapping("download/jar")
public ResponseEntity<byte[]> downloadJmeterFiles(@RequestBody Map<String, List<ProjectJarConfig>> jarConfigs) {
byte[] bytes = apiJmeterFileService.downloadJmeterJar(jarConfigs);
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType("application/octet-stream"))
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + UUID.randomUUID().toString() + ".zip\"")

29
api-test/backend/src/main/java/io/metersphere/service/ApiJMeterFileService.java
View File

@ -16,6 +16,7 @@ import io.metersphere.commons.constants.PluginScenario;
import io.metersphere.commons.utils.*;
import io.metersphere.dto.AttachmentBodyFile;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.dto.ProjectJarConfig;
import io.metersphere.environment.service.BaseEnvGroupProjectService;
import io.metersphere.metadata.service.FileMetadataService;
import io.metersphere.request.BodyFile;
@ -152,22 +153,32 @@ public class ApiJMeterFileService {
return listBytesToZip(files);
}
public byte[] downloadJmeterJar(String projectId) {
public byte[] downloadJmeterJar(Map<String, List<ProjectJarConfig>> map) {
Map<String, byte[]> files = new HashMap<>();
// 获取JAR
Map<String, byte[]> jarFiles = this.getJar(projectId);
if (!MapUtils.isEmpty(jarFiles)) {
for (String k : jarFiles.keySet()) {
byte[] v = jarFiles.get(k);
files.put(k, v);
}
if (MapUtils.isNotEmpty(map)) {
//获取文件内容
FileMetadataService fileMetadataService = CommonBeanFactory.getBean(FileMetadataService.class);
map.forEach((key, value) -> {
//历史数据
value.forEach(s -> {
//获取文件内容;
// 兼容历史数据
byte[] bytes = fileMetadataService.getContent(s.getId());
files.put(StringUtils.join(
key,
File.separator,
s.getId(),
File.separator,
String.valueOf(s.getUpdateTime()), ".jar"), bytes);
});
});
}
return listBytesToZip(files);
}
public byte[] downloadPluginJar(List<String> pluginIds) {
Map<String, byte[]> files = new HashMap<>();
if (!CollectionUtils.isNotEmpty(pluginIds)) {
if (CollectionUtils.isNotEmpty(pluginIds)) {
// 获取JAR
Map<String, byte[]> jarFiles = this.getPlugJar(pluginIds);
if (MapUtils.isNotEmpty(jarFiles)) {

4
api-test/backend/src/main/java/io/metersphere/service/scenario/ApiScenarioService.java
View File

@ -15,7 +15,6 @@ import io.metersphere.api.dto.scenario.ApiScenarioParamDTO;
import io.metersphere.api.exec.scenario.ApiScenarioEnvService;
import io.metersphere.api.exec.scenario.ApiScenarioExecuteService;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.api.jmeter.NewDriverManager;
import io.metersphere.api.parse.ApiImportParser;
import io.metersphere.api.parse.scenario.ApiScenarioImportUtil;
import io.metersphere.api.parse.scenario.ScenarioImport;
@ -57,6 +56,7 @@ import io.metersphere.service.definition.TcpApiParamService;
import io.metersphere.service.ext.ExtApiScheduleService;
import io.metersphere.service.ext.ExtFileAssociationService;
import io.metersphere.service.plan.TestPlanScenarioCaseService;
import jakarta.annotation.Resource;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.BooleanUtils;
@ -76,7 +76,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import jakarta.annotation.Resource;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
@ -851,7 +850,6 @@ public class ApiScenarioService {
String projectId = apiScenario.getProjectId();
List<String> projectIds = new ArrayList<>();
projectIds.add(projectId);
testPlan.setJarPaths(NewDriverManager.getJars(projectIds));
testPlan.setName(apiScenario.getName());
testPlan.setHashTree(new LinkedList<>());
ParameterConfig config = new ParameterConfig();

4
framework/sdk-parent/jmeter/pom.xml
View File

@ -290,6 +290,10 @@
<artifactId>jorphan</artifactId>
<version>${jmeter.version}</version>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
</dependency>
</dependencies>

16
framework/sdk-parent/jmeter/src/main/java/io/metersphere/dto/ProjectJarConfig.java Normal file
View File

@ -0,0 +1,16 @@
package io.metersphere.dto;
import lombok.Data;
@Data
public class ProjectJarConfig {
private String id;
private String name;
private long updateTime;
// 是否已有文件
private boolean hasFile;
// 文件存储类型
private String storage;
private String attachInfo;
}

104
framework/sdk-parent/jmeter/src/main/java/io/metersphere/utils/JarConfigUtils.java Normal file
View File

@ -0,0 +1,104 @@
package io.metersphere.utils;
import io.metersphere.dto.ProjectJarConfig;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.util.FileUtil;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class JarConfigUtils {
public static Map<String, List<ProjectJarConfig>> getJarConfigs(List<String> projectIds, Map<String, List<ProjectJarConfig>> jarConfigMap) {
String localPath = LocalPathUtil.prePath;
Map<String, List<ProjectJarConfig>> jarConfigsMap = new HashMap<>();
projectIds.forEach(item -> {
List<ProjectJarConfig> jarConfigs = new ArrayList<>();
//根据项目id获取node服务器当前项目下的所有文件
List<String> nodeFiles = getFileNames(StringUtils.join(localPath, File.separator, item));
//主服务的jar文件
if (jarConfigMap.containsKey(item)) {
List<ProjectJarConfig> projectJarConfigs = jarConfigMap.get(item);
//本地文件多余的jar进行删除
List<String> expiredJar = nodeFiles
.stream().filter(nodeFile -> !projectJarConfigs
.stream()
.map(ProjectJarConfig::getId)
.collect(Collectors.toList()).contains(nodeFile)).collect(Collectors.toList());
expiredJar.forEach(jar
-> deleteDir(StringUtils.join(
localPath,
File.separator,
item,
File.separator,
jar)));
projectJarConfigs.forEach(projectJarConfig -> {
if (CollectionUtils.isNotEmpty(nodeFiles)) {
nodeFiles.forEach(refId -> {
if (!nodeFiles.contains(projectJarConfig.getId())) {
jarConfigs.add(projectJarConfig);
} else if (StringUtils.equals(refId, projectJarConfig.getId())) {
//资源id目录存在，需要判断文件的时间戳与数据记录的时间戳是不是同一个 不是同一个的需要删除jar然后重新下载
List<String> jarFiles = getFileNames(StringUtils.join(localPath, File.separator, item, File.separator, projectJarConfig.getId()));
if (CollectionUtils.isNotEmpty(jarFiles)) {
jarFiles.forEach(jarfile -> {
long updateTime = projectJarConfig.getUpdateTime();
if (!StringUtils.equals(StringUtils.substringBefore(jarfile, ".jar"), String.valueOf(updateTime))) {
deleteFile(StringUtils.join(localPath, File.separator, item, File.separator, refId, File.separator, jarfile));
jarConfigs.add(projectJarConfig);
}
});
} else {
jarConfigs.add(projectJarConfig);
}
}
});
} else {
//本地没有文件，需要从服务器下载
jarConfigs.add(projectJarConfig);
}
if (CollectionUtils.isNotEmpty(jarConfigs)) {
jarConfigsMap.put(item, jarConfigs);
}
});
}
});
return jarConfigsMap;
}
public static void deleteFile(String path) {
File file = new File(path);
if (file.exists()) {
file.delete();
}
}
public static List<String> getFileNames(String path) {
File f = new File(path);
if (!f.exists()) {
f.mkdirs();
}
List<String> fileNames = new ArrayList<>();
File fa[] = f.listFiles();
for (int i = 0; i < fa.length; i++) {
File fs = fa[i];
if (fs.exists()) {
fileNames.add(fs.getName());
}
}
return fileNames;
}
public static void deleteDir(String path) {
File file = new File(path);
FileUtil.deleteContents(file);
if (file.exists()) {
file.delete();
}
}
}

11
framework/sdk-parent/jmeter/src/main/java/io/metersphere/utils/JsonUtils.java
View File

@ -2,6 +2,7 @@ package io.metersphere.utils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
@ -32,7 +33,7 @@ public class JsonUtils {
}
public static Object parseObject(String content) {
return parseObject(content, Object.class);
return parseObject(content, Object.class);
}
public static <T> T parseObject(String content, Class<T> valueType) {
@ -51,6 +52,14 @@ public class JsonUtils {
}
}
public static <T> T parseObject(String content, TypeReference<T> valueType) {
try {
return objectMapper.readValue(content, valueType);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static List parseArray(String content) {
return parseArray(content, Object.class);
}

5
framework/sdk-parent/jmeter/src/main/java/io/metersphere/utils/LocalPathUtil.java Normal file
View File

@ -0,0 +1,5 @@
package io.metersphere.utils;
public class LocalPathUtil {
public static String prePath;
}

1
framework/sdk-parent/sdk/src/main/java/io/metersphere/commons/utils/FileUtils.java
View File

@ -33,6 +33,7 @@ public class FileUtils {
public static final String UI_IMAGE_DIR = "/opt/metersphere/data/image/ui/screenshots";
public static final String ATTACHMENT_DIR = "/opt/metersphere/data/attachment";
public static final String ATTACHMENT_TMP_DIR = "/opt/metersphere/data/attachment/tmp";
public static final String LOCAL_JAR = "/opt/metersphere/data/local-jar/jar";
public static void validateFileName(String fileName) {
if (StringUtils.isNotEmpty(fileName) && StringUtils.contains(fileName, "." + File.separator)) {

40
framework/sdk-parent/sdk/src/main/java/io/metersphere/metadata/service/FileMetadataService.java
View File

@ -561,6 +561,25 @@ public class FileMetadataService {
return fileInfoDTOList;
}
public List<FileInfoDTO> downloadFileByIds(Collection<String> fileIdList) {
if (CollectionUtils.isEmpty(fileIdList)) {
return new ArrayList<>(0);
}
LogUtil.info(JSON.toJSONString(fileIdList) + " 获取文件开始");
FileMetadataExample example = new FileMetadataExample();
example.createCriteria().andIdIn(new ArrayList<>(fileIdList));
List<FileMetadataWithBLOBs> fileMetadataWithBLOBs = fileMetadataMapper.selectByExampleWithBLOBs(example);
List<FileRequest> requestList = new ArrayList<>();
fileMetadataWithBLOBs.forEach(fileMetadata -> {
requestList.add(this.genFileRequest(fileMetadata));
});
List<FileInfoDTO> repositoryFileDTOList = fileManagerService.downloadFileBatch(requestList);
LogUtil.info(JSON.toJSONString(fileIdList) + " 获取文件结束。");
return repositoryFileDTOList;
}
private FileRequest genFileRequest(FileMetadataWithBLOBs fileMetadata) {
if (fileMetadata != null) {
FileRequest request = new FileRequest(fileMetadata.getProjectId(), fileMetadata.getName(), fileMetadata.getType());
@ -582,26 +601,7 @@ public class FileMetadataService {
return new FileRequest();
}
}
public List<FileInfoDTO> downloadFileByIds(Collection<String> fileIdList) {
if (CollectionUtils.isEmpty(fileIdList)) {
return new ArrayList<>(0);
}
LogUtil.info(JSON.toJSONString(fileIdList) + " 获取文件开始");
FileMetadataExample example = new FileMetadataExample();
example.createCriteria().andIdIn(new ArrayList<>(fileIdList));
List<FileMetadataWithBLOBs> fileMetadataWithBLOBs = fileMetadataMapper.selectByExampleWithBLOBs(example);
List<FileRequest> requestList = new ArrayList<>();
fileMetadataWithBLOBs.forEach(fileMetadata -> {
requestList.add(this.genFileRequest(fileMetadata));
});
List<FileInfoDTO> repositoryFileDTOList = fileManagerService.downloadFileBatch(requestList);
LogUtil.info(JSON.toJSONString(fileIdList) + " 获取文件结束。");
return repositoryFileDTOList;
}
public FileMetadata pullFromRepository(FileMetadata request) {
FileMetadata returnModel = null;
FileMetadataWithBLOBs baseMetadata = fileMetadataMapper.selectByPrimaryKey(request.getId());