test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(接口测试): 接口测试大量数据的导出优化

This commit is contained in:
Jianguo-Genius 2024-09-25 19:10:09 +08:00 committed by Craftsman
parent 2a35036652
commit 26c75fe483
7 changed files with 145 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/framework/sdk/src/main/java/io/metersphere/sdk/constants/DefaultRepositoryDir.java
View File

@ -33,6 +33,7 @@ public class DefaultRepositoryDir {
*/
private static final String SYSTEM_TEMP_DIR = SYSTEM_ROOT_DIR + "/temp";
private static final String EXPORT_EXCEL_TEMP_DIR = SYSTEM_ROOT_DIR + "/export/excel";
private static final String EXPORT_API_TEMP_DIR = SYSTEM_ROOT_DIR + "/export/api";
/*------ end: 系统下资源目录 --------*/
@ -162,6 +163,10 @@ public class DefaultRepositoryDir {
public static String getExportExcelTempDir() {
return EXPORT_EXCEL_TEMP_DIR;
}
public static String getExportApiTempDir() {
return EXPORT_API_TEMP_DIR;
}
public static String getSystemTempCompressDir() {
return SYSTEM_TEMP_DIR + "/compress";
}

88
backend/framework/sdk/src/main/java/io/metersphere/sdk/util/MsFileUtils.java
View File

@ -5,11 +5,12 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.*;
import java.util.Objects;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
public class MsFileUtils {
public static void validateFileName(String... fileNames) {
@ -60,4 +61,83 @@ public class MsFileUtils {
}
return null;
}
public static File zipFile(String rootPath, String zipFolder) {
File folder = new File(rootPath + File.separator + zipFolder);
if (folder.isDirectory()) {
File[] files = folder.listFiles();
if (files == null || files.length == 0) {
return null;
}
File zipFile = new File(rootPath + File.separator + zipFolder + ".zip");
try (ZipOutputStream zipOutputStream = new ZipOutputStream(new FileOutputStream(zipFile))) {
for (File file : files) {
String fileName = file.getName();
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) {
zipOutputStream.putNextEntry(new ZipEntry(fileName));
byte[] buffer = new byte[512];
int num;
while ((num = bis.read(buffer)) > 0) {
zipOutputStream.write(buffer, 0, num);
}
zipOutputStream.closeEntry();
} catch (Exception ignore) {
}
}
} catch (Exception e) {
LogUtils.error(e);
}
return zipFile;
}
return null;
}
public static File[] unZipFile(File file, String targetPath) {
InputStream input = null;
OutputStream output = null;
try (ZipInputStream zipInput = new ZipInputStream(new FileInputStream(file));
ZipFile zipFile = new ZipFile(file)) {
ZipEntry entry = null;
while ((entry = zipInput.getNextEntry()) != null) {
File outFile = new File(targetPath + File.separator + entry.getName());
if (!outFile.getParentFile().exists()) {
outFile.getParentFile().mkdir();
}
if (!outFile.exists()) {
outFile.createNewFile();
}
input = zipFile.getInputStream(entry);
output = new FileOutputStream(outFile);
int temp = 0;
while ((temp = input.read()) != -1) {
output.write(temp);
}
}
File folder = new File(targetPath);
if (folder.isDirectory()) {
return folder.listFiles();
} else {
return null;
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (input != null) {
try {
input.close();
} catch (Exception ignore) {
}
}
if (output != null) {
try {
output.close();
} catch (Exception ignore) {
}
}
}
return null;
}
}

2
backend/services/api-test/src/main/java/io/metersphere/api/controller/definition/ApiDefinitionController.java
View File

@ -314,7 +314,7 @@ public class ApiDefinitionController {
@PostMapping(value = "/export/{type}")
@Operation(summary = "接口测试-接口管理-导出接口定义")
@RequiresPermissions(PermissionConstants.PROJECT_API_DEFINITION_EXPORT)
public String newExport(@RequestBody ApiDefinitionBatchExportRequest request, @PathVariable String type) {
public String export(@RequestBody ApiDefinitionBatchExportRequest request, @PathVariable String type) {
return apiDefinitionExportService.exportApiDefinition(request, type, SessionUtils.getUserId());
}

53
backend/services/api-test/src/main/java/io/metersphere/api/service/definition/ApiDefinitionExportService.java
View File

@ -16,10 +16,7 @@ import io.metersphere.sdk.constants.*;
import io.metersphere.sdk.dto.ExportMsgDTO;
import io.metersphere.sdk.exception.MSException;
import io.metersphere.sdk.file.FileRequest;
import io.metersphere.sdk.util.JSON;
import io.metersphere.sdk.util.LogUtils;
import io.metersphere.sdk.util.MsFileUtils;
import io.metersphere.sdk.util.Translator;
import io.metersphere.sdk.util.*;
import io.metersphere.system.constants.ExportConstants;
import io.metersphere.system.domain.User;
import io.metersphere.system.dto.sdk.BaseTreeNode;
@ -89,14 +86,9 @@ public class ApiDefinitionExportService {
});
return modulePathMap;
}
public ApiExportResponse genApiExportResponse(ApiDefinitionBatchExportRequest request, String type, String userId) {
List<String> ids = this.getBatchExportApiIds(request, request.getProjectId(), userId);
if (CollectionUtils.isEmpty(ids)) {
return null;
}
List<ApiDefinitionWithBlob> list = this.selectAndSortByIds(ids);
List<String> moduleIds = list.stream().map(ApiDefinitionWithBlob::getModuleId).toList();
Map<String, String> moduleMap = this.buildModuleIdPathMap(request.getProjectId());
public ApiExportResponse genApiExportResponse(ApiDefinitionBatchExportRequest request, Map<String, String> moduleMap, String type, String userId) {
List<ApiDefinitionWithBlob> list = this.selectAndSortByIds(request.getSelectIds());
return switch (type.toLowerCase()) {
case "swagger" -> exportSwagger(request, list, moduleMap);
case "metersphere" -> exportMetersphere(request, list, moduleMap);
@ -119,7 +111,7 @@ public class ApiDefinitionExportService {
}
});
returnId = exportTask.getId();
} catch (InterruptedException e) {
} catch (Exception e) {
LogUtils.error("导出失败：" + e);
throw new MSException(e);
}
@ -129,10 +121,10 @@ public class ApiDefinitionExportService {
@Resource
private FileService fileService;
public void uploadFileToMinio(String fileType, File file, String fileId) throws Exception {
public void uploadFileToMinioExportFolder(File file, String fileName) throws Exception {
FileRequest fileRequest = new FileRequest();
fileRequest.setFileName(fileId.concat(".").concat(fileType));
fileRequest.setFolder(DefaultRepositoryDir.getExportExcelTempDir());
fileRequest.setFileName(fileName);
fileRequest.setFolder(DefaultRepositoryDir.getExportApiTempDir());
fileRequest.setStorage(StorageType.MINIO.name());
try (FileInputStream inputStream = new FileInputStream(file)) {
fileService.upload(inputStream, fileRequest);
@ -156,18 +148,35 @@ public class ApiDefinitionExportService {
if (CollectionUtils.isEmpty(ids)) {
return null;
}
ApiExportResponse exportResponse = this.genApiExportResponse(request, exportType, userId);
File createFile = new File(tmpDir.getPath() + File.separatorChar + request.getFileId() + ".json");
Map<String, String> moduleMap = this.buildModuleIdPathMap(request.getProjectId());
String fileFolder = tmpDir.getPath() + File.separatorChar + request.getFileId();
int fileIndex = 1;
SubListUtils.dealForSubList(ids, 1000, subList -> {
request.setSelectIds(subList);
ApiExportResponse exportResponse = this.genApiExportResponse(request, moduleMap, exportType, userId);
File createFile = new File(fileFolder + File.separatorChar + fileIndex + ".json");
if (!createFile.exists()) {
try {
createFile.getParentFile().mkdirs();
createFile.createNewFile();
} catch (IOException e) {
throw new MSException(e);
}
}
try {
FileUtils.writeByteArrayToFile(createFile, JSON.toJSONString(exportResponse).getBytes());
fileType = "json";
uploadFileToMinio(fileType, createFile, request.getFileId());
} catch (Exception e) {
LogUtils.error(e);
}
});
File zipFile = MsFileUtils.zipFile(tmpDir.getPath(), request.getFileId());
if (zipFile == null) {
return null;
}
uploadFileToMinioExportFolder(zipFile, request.getFileId());
// 生成日志
LogDTO logDTO = apiDefinitionLogService.exportExcelLog(request, exportType, userId, projectMapper.selectByPrimaryKey(request.getProjectId()).getOrganizationId());
@ -261,8 +270,8 @@ public class ApiDefinitionExportService {
ExportTask tasksFirst = exportTasks.getFirst();
Project project = projectMapper.selectByPrimaryKey(projectId);
FileRequest fileRequest = new FileRequest();
fileRequest.setFileName(tasksFirst.getFileId().concat(".").concat("json"));
fileRequest.setFolder(DefaultRepositoryDir.getExportExcelTempDir());
fileRequest.setFileName(tasksFirst.getFileId());
fileRequest.setFolder(DefaultRepositoryDir.getExportApiTempDir());
fileRequest.setStorage(StorageType.MINIO.name());
String fileName = "Metersphere_case_" + project.getName() + "." + tasksFirst.getFileType();
try {

10
backend/services/api-test/src/test/java/io/metersphere/api/controller/ApiDefinitionControllerTests.java
View File

@ -2099,15 +2099,19 @@ public class ApiDefinitionControllerTests extends BaseTest {
byte[] fileBytes = mvcResult.getResponse().getContentAsByteArray();
File file = new File("/tmp/test.json");
FileUtils.writeByteArrayToFile(file, fileBytes);
File zipFile = new File("/tmp/api-export/downloadFiles.zip");
FileUtils.writeByteArrayToFile(zipFile, fileBytes);
String fileContent = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
File[] files = MsFileUtils.unZipFile(zipFile, "/tmp/api-export/unzip/");
assert files != null;
Assertions.assertEquals(files.length, 1);
String fileContent = FileUtils.readFileToString(files[0], StandardCharsets.UTF_8);
MetersphereApiExportResponse exportResponse = ApiDataUtils.parseObject(fileContent, MetersphereApiExportResponse.class);
apiDefinitionImportTestService.compareApiExport(exportResponse, exportApiBlobs);
MsFileUtils.deleteDir("/tmp/api-export/");
//测试stop
this.requestGetWithOk("/stop/" + taskId);

2
backend/services/project-management/src/main/java/io/metersphere/project/utils/FileDownloadUtils.java
View File

@ -49,7 +49,7 @@ public class FileDownloadUtils {
outputStream.write(buffer, 0, num);
}
outputStream.close();
response.setContentType("application/zip");
response.setContentType("application/octet-stream");
response.setCharacterEncoding(StandardCharsets.UTF_8.name());
response.setHeader("Content-disposition", "attachment;filename=" + fileName);
} catch (Exception e) {

10
backend/services/system-setting/src/main/java/io/metersphere/system/config/MinioConfig.java
View File

@ -87,6 +87,16 @@ public class MinioConfig {
null,
null,
null));
rules.add(
new LifecycleRule(
Status.ENABLED,
null,
new Expiration((ZonedDateTime) null, 1, null),
new RuleFilter("system/export/api"),
"api-file",
null,
null,
null));
LifecycleConfiguration config = new LifecycleConfiguration(rules);
try {
minioClient.setBucketLifecycle(