refactor(接口测试): 接口测试大量数据的导出优化

backend/framework/sdk/src/main/java/io/metersphere/sdk/constants/DefaultRepositoryDir.java

@@ -33,6 +33,7 @@ public class DefaultRepositoryDir {
      */
     private static final String SYSTEM_TEMP_DIR = SYSTEM_ROOT_DIR + "/temp";
     private static final String EXPORT_EXCEL_TEMP_DIR = SYSTEM_ROOT_DIR + "/export/excel";
+    private static final String EXPORT_API_TEMP_DIR = SYSTEM_ROOT_DIR + "/export/api";
 
     /*------ end: 系统下资源目录 --------*/
 
@@ -162,6 +163,10 @@ public class DefaultRepositoryDir {
     public static String getExportExcelTempDir() {
         return EXPORT_EXCEL_TEMP_DIR;
     }
+
+    public static String getExportApiTempDir() {
+        return EXPORT_API_TEMP_DIR;
+    }
     public static String getSystemTempCompressDir() {
         return SYSTEM_TEMP_DIR + "/compress";
     }

backend/framework/sdk/src/main/java/io/metersphere/sdk/util/MsFileUtils.java

@@ -5,11 +5,12 @@ import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.web.multipart.MultipartFile;
 
-import java.io.File;
+import java.io.*;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.util.Objects;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
 
 public class MsFileUtils {
     public static void validateFileName(String... fileNames) {
@@ -60,4 +61,83 @@ public class MsFileUtils {
         }
         return null;
     }
+
+    public static File zipFile(String rootPath, String zipFolder) {
+        File folder = new File(rootPath + File.separator + zipFolder);
+        if (folder.isDirectory()) {
+            File[] files = folder.listFiles();
+
+            if (files == null || files.length == 0) {
+                return null;
+            }
+            File zipFile = new File(rootPath + File.separator + zipFolder + ".zip");
+
+            try (ZipOutputStream zipOutputStream = new ZipOutputStream(new FileOutputStream(zipFile))) {
+                for (File file : files) {
+                    String fileName = file.getName();
+                    try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) {
+                        zipOutputStream.putNextEntry(new ZipEntry(fileName));
+                        byte[] buffer = new byte[512];
+                        int num;
+                        while ((num = bis.read(buffer)) > 0) {
+                            zipOutputStream.write(buffer, 0, num);
+                        }
+                        zipOutputStream.closeEntry();
+                    } catch (Exception ignore) {
+                    }
+                }
+            } catch (Exception e) {
+                LogUtils.error(e);
+            }
+            return zipFile;
+        }
+
+        return null;
+    }
+
+    public static File[] unZipFile(File file, String targetPath) {
+        InputStream input = null;
+        OutputStream output = null;
+        try (ZipInputStream zipInput = new ZipInputStream(new FileInputStream(file));
+             ZipFile zipFile = new ZipFile(file)) {
+            ZipEntry entry = null;
+            while ((entry = zipInput.getNextEntry()) != null) {
+                File outFile = new File(targetPath + File.separator + entry.getName());
+                if (!outFile.getParentFile().exists()) {
+                    outFile.getParentFile().mkdir();
+                }
+                if (!outFile.exists()) {
+                    outFile.createNewFile();
+                }
+                input = zipFile.getInputStream(entry);
+                output = new FileOutputStream(outFile);
+                int temp = 0;
+                while ((temp = input.read()) != -1) {
+                    output.write(temp);
+                }
+            }
+            File folder = new File(targetPath);
+            if (folder.isDirectory()) {
+                return folder.listFiles();
+            } else {
+                return null;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            if (input != null) {
+                try {
+                    input.close();
+                } catch (Exception ignore) {
+                }
+            }
+            if (output != null) {
+                try {
+                    output.close();
+                } catch (Exception ignore) {
+                }
+            }
+        }
+        return null;
+    }
 }

backend/services/api-test/src/main/java/io/metersphere/api/controller/definition/ApiDefinitionController.java

@@ -314,7 +314,7 @@ public class ApiDefinitionController {
     @PostMapping(value = "/export/{type}")
     @Operation(summary = "接口测试-接口管理-导出接口定义")
     @RequiresPermissions(PermissionConstants.PROJECT_API_DEFINITION_EXPORT)
-    public String newExport(@RequestBody ApiDefinitionBatchExportRequest request, @PathVariable String type) {
+    public String export(@RequestBody ApiDefinitionBatchExportRequest request, @PathVariable String type) {
         return apiDefinitionExportService.exportApiDefinition(request, type, SessionUtils.getUserId());
     }
 

backend/services/api-test/src/main/java/io/metersphere/api/service/definition/ApiDefinitionExportService.java

@@ -16,10 +16,7 @@ import io.metersphere.sdk.constants.*;
 import io.metersphere.sdk.dto.ExportMsgDTO;
 import io.metersphere.sdk.exception.MSException;
 import io.metersphere.sdk.file.FileRequest;
-import io.metersphere.sdk.util.JSON;
+import io.metersphere.sdk.util.*;
-import io.metersphere.sdk.util.LogUtils;
-import io.metersphere.sdk.util.MsFileUtils;
-import io.metersphere.sdk.util.Translator;
 import io.metersphere.system.constants.ExportConstants;
 import io.metersphere.system.domain.User;
 import io.metersphere.system.dto.sdk.BaseTreeNode;
@@ -89,14 +86,9 @@ public class ApiDefinitionExportService {
         });
         return modulePathMap;
     }
-    public ApiExportResponse genApiExportResponse(ApiDefinitionBatchExportRequest request, String type, String userId) {
+
-        List<String> ids = this.getBatchExportApiIds(request, request.getProjectId(), userId);
+    public ApiExportResponse genApiExportResponse(ApiDefinitionBatchExportRequest request, Map<String, String> moduleMap, String type, String userId) {
-        if (CollectionUtils.isEmpty(ids)) {
+        List<ApiDefinitionWithBlob> list = this.selectAndSortByIds(request.getSelectIds());
-            return null;
-        }
-        List<ApiDefinitionWithBlob> list = this.selectAndSortByIds(ids);
-        List<String> moduleIds = list.stream().map(ApiDefinitionWithBlob::getModuleId).toList();
-        Map<String, String> moduleMap = this.buildModuleIdPathMap(request.getProjectId());
         return switch (type.toLowerCase()) {
             case "swagger" -> exportSwagger(request, list, moduleMap);
             case "metersphere" -> exportMetersphere(request, list, moduleMap);
@@ -119,7 +111,7 @@ public class ApiDefinitionExportService {
                         }
                     });
             returnId = exportTask.getId();
-        } catch (InterruptedException e) {
+        } catch (Exception e) {
             LogUtils.error("导出失败：" + e);
             throw new MSException(e);
         }
@@ -129,10 +121,10 @@ public class ApiDefinitionExportService {
     @Resource
     private FileService fileService;
 
-    public void uploadFileToMinio(String fileType, File file, String fileId) throws Exception {
+    public void uploadFileToMinioExportFolder(File file, String fileName) throws Exception {
         FileRequest fileRequest = new FileRequest();
-        fileRequest.setFileName(fileId.concat(".").concat(fileType));
+        fileRequest.setFileName(fileName);
-        fileRequest.setFolder(DefaultRepositoryDir.getExportExcelTempDir());
+        fileRequest.setFolder(DefaultRepositoryDir.getExportApiTempDir());
         fileRequest.setStorage(StorageType.MINIO.name());
         try (FileInputStream inputStream = new FileInputStream(file)) {
             fileService.upload(inputStream, fileRequest);
@@ -156,18 +148,35 @@ public class ApiDefinitionExportService {
             if (CollectionUtils.isEmpty(ids)) {
                 return null;
             }
-            ApiExportResponse exportResponse = this.genApiExportResponse(request, exportType, userId);
+
-            File createFile = new File(tmpDir.getPath() + File.separatorChar + request.getFileId() + ".json");
+            Map<String, String> moduleMap = this.buildModuleIdPathMap(request.getProjectId());
+
+            String fileFolder = tmpDir.getPath() + File.separatorChar + request.getFileId();
+            int fileIndex = 1;
+            SubListUtils.dealForSubList(ids, 1000, subList -> {
+                request.setSelectIds(subList);
+                ApiExportResponse exportResponse = this.genApiExportResponse(request, moduleMap, exportType, userId);
+                File createFile = new File(fileFolder + File.separatorChar + fileIndex + ".json");
                 if (!createFile.exists()) {
                     try {
+                        createFile.getParentFile().mkdirs();
                         createFile.createNewFile();
                     } catch (IOException e) {
                         throw new MSException(e);
                     }
                 }
+                try {
                     FileUtils.writeByteArrayToFile(createFile, JSON.toJSONString(exportResponse).getBytes());
-            fileType = "json";
+                } catch (Exception e) {
-            uploadFileToMinio(fileType, createFile, request.getFileId());
+                    LogUtils.error(e);
+                }
+            });
+            File zipFile = MsFileUtils.zipFile(tmpDir.getPath(), request.getFileId());
+            if (zipFile == null) {
+                return null;
+            }
+
+            uploadFileToMinioExportFolder(zipFile, request.getFileId());
 
             // 生成日志
             LogDTO logDTO = apiDefinitionLogService.exportExcelLog(request, exportType, userId, projectMapper.selectByPrimaryKey(request.getProjectId()).getOrganizationId());
@@ -261,8 +270,8 @@ public class ApiDefinitionExportService {
         ExportTask tasksFirst = exportTasks.getFirst();
         Project project = projectMapper.selectByPrimaryKey(projectId);
         FileRequest fileRequest = new FileRequest();
-        fileRequest.setFileName(tasksFirst.getFileId().concat(".").concat("json"));
+        fileRequest.setFileName(tasksFirst.getFileId());
-        fileRequest.setFolder(DefaultRepositoryDir.getExportExcelTempDir());
+        fileRequest.setFolder(DefaultRepositoryDir.getExportApiTempDir());
         fileRequest.setStorage(StorageType.MINIO.name());
         String fileName = "Metersphere_case_" + project.getName() + "." + tasksFirst.getFileType();
         try {

backend/services/api-test/src/test/java/io/metersphere/api/controller/ApiDefinitionControllerTests.java

@@ -2099,15 +2099,19 @@ public class ApiDefinitionControllerTests extends BaseTest {
 
         byte[] fileBytes = mvcResult.getResponse().getContentAsByteArray();
 
-        File file = new File("/tmp/test.json");
+        File zipFile = new File("/tmp/api-export/downloadFiles.zip");
-        FileUtils.writeByteArrayToFile(file, fileBytes);
+        FileUtils.writeByteArrayToFile(zipFile, fileBytes);
 
-        String fileContent = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
+        File[] files = MsFileUtils.unZipFile(zipFile, "/tmp/api-export/unzip/");
+        assert files != null;
+        Assertions.assertEquals(files.length, 1);
+        String fileContent = FileUtils.readFileToString(files[0], StandardCharsets.UTF_8);
 
         MetersphereApiExportResponse exportResponse = ApiDataUtils.parseObject(fileContent, MetersphereApiExportResponse.class);
 
         apiDefinitionImportTestService.compareApiExport(exportResponse, exportApiBlobs);
 
+        MsFileUtils.deleteDir("/tmp/api-export/");
 
         //测试stop
         this.requestGetWithOk("/stop/" + taskId);

backend/services/project-management/src/main/java/io/metersphere/project/utils/FileDownloadUtils.java

@@ -49,7 +49,7 @@ public class FileDownloadUtils {
                 outputStream.write(buffer, 0, num);
             }
             outputStream.close();
-            response.setContentType("application/zip");
+            response.setContentType("application/octet-stream");
             response.setCharacterEncoding(StandardCharsets.UTF_8.name());
             response.setHeader("Content-disposition", "attachment;filename=" + fileName);
         } catch (Exception e) {

backend/services/system-setting/src/main/java/io/metersphere/system/config/MinioConfig.java

@@ -87,6 +87,16 @@ public class MinioConfig {
                         null,
                         null,
                         null));
+        rules.add(
+                new LifecycleRule(
+                        Status.ENABLED,
+                        null,
+                        new Expiration((ZonedDateTime) null, 1, null),
+                        new RuleFilter("system/export/api"),
+                        "api-file",
+                        null,
+                        null,
+                        null));
         LifecycleConfiguration config = new LifecycleConfiguration(rules);
         try {
             minioClient.setBucketLifecycle(