From be95c69af599e1de4a6ff74b804e9d511b685e74 Mon Sep 17 00:00:00 2001 From: "Captain.B" Date: Mon, 21 Dec 2020 14:13:54 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=E9=87=8D=E6=9E=84=E4=BC=A0?= =?UTF-8?q?=E9=80=92=E6=B5=8B=E8=AF=95=E6=96=87=E4=BB=B6=E7=9A=84=E6=96=B9?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/pom.xml | 6 + .../metersphere/config/JmeterProperties.java | 1 + .../io/metersphere/config/ShiroConfig.java | 1 + .../controller/JmeterFileController.java | 33 +++++ .../performance/engine/EngineContext.java | 9 -- .../performance/engine/EngineFactory.java | 14 -- .../engine/docker/DockerTestEngine.java | 60 ++++----- .../engine/docker/request/BaseRequest.java | 11 -- .../engine/docker/request/TestRequest.java | 18 --- .../engine/request/BaseRequest.java | 9 ++ .../engine/request/StartTestRequest.java | 14 ++ .../engine/request/StopTestRequest.java | 9 ++ .../service/JmeterFileService.java | 121 ++++++++++++++++++ backend/src/main/java/io/metersphere/xpack | 2 +- 14 files changed, 226 insertions(+), 82 deletions(-) create mode 100644 backend/src/main/java/io/metersphere/performance/controller/JmeterFileController.java delete mode 100644 backend/src/main/java/io/metersphere/performance/engine/docker/request/BaseRequest.java delete mode 100644 backend/src/main/java/io/metersphere/performance/engine/docker/request/TestRequest.java create mode 100644 backend/src/main/java/io/metersphere/performance/engine/request/BaseRequest.java create mode 100644 backend/src/main/java/io/metersphere/performance/engine/request/StartTestRequest.java create mode 100644 backend/src/main/java/io/metersphere/performance/engine/request/StopTestRequest.java create mode 100644 backend/src/main/java/io/metersphere/performance/service/JmeterFileService.java diff --git a/backend/pom.xml b/backend/pom.xml index 961d3aab62..7c1e20dd8e 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -379,6 +379,12 @@ reflections8 0.11.7 + + + io.fabric8 + kubernetes-client + 4.13.0 + diff --git a/backend/src/main/java/io/metersphere/config/JmeterProperties.java b/backend/src/main/java/io/metersphere/config/JmeterProperties.java index bfe302b5fd..af59bb8fb2 100644 --- a/backend/src/main/java/io/metersphere/config/JmeterProperties.java +++ b/backend/src/main/java/io/metersphere/config/JmeterProperties.java @@ -16,4 +16,5 @@ public class JmeterProperties { private String home; + private String heap = "-Xms1g -Xmx1g -XX:MaxMetaspaceSize=256m"; } diff --git a/backend/src/main/java/io/metersphere/config/ShiroConfig.java b/backend/src/main/java/io/metersphere/config/ShiroConfig.java index ddc9ad8e2c..1fa1508bfe 100644 --- a/backend/src/main/java/io/metersphere/config/ShiroConfig.java +++ b/backend/src/main/java/io/metersphere/config/ShiroConfig.java @@ -45,6 +45,7 @@ public class ShiroConfig implements EnvironmentAware { ShiroUtils.loadBaseFilterChain(filterChainDefinitionMap); filterChainDefinitionMap.put("/display/info", "anon"); filterChainDefinitionMap.put("/display/file/**", "anon"); + filterChainDefinitionMap.put("/jmeter/download/**", "anon"); filterChainDefinitionMap.put("/**", "apikey, authc"); return shiroFilterFactoryBean; } diff --git a/backend/src/main/java/io/metersphere/performance/controller/JmeterFileController.java b/backend/src/main/java/io/metersphere/performance/controller/JmeterFileController.java new file mode 100644 index 0000000000..b4f66783f0 --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/controller/JmeterFileController.java @@ -0,0 +1,33 @@ +package io.metersphere.performance.controller; + + +import io.metersphere.performance.service.JmeterFileService; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; + +@RestController +@RequestMapping("jmeter") +public class JmeterFileController { + @Resource + private JmeterFileService jmeterFileService; + + @GetMapping("download") + public ResponseEntity downloadJmeterFiles(@RequestParam("testId") String testId, @RequestParam("resourceId") String resourceId, + @RequestParam("ratio") double ratio, @RequestParam("startTime") long startTime, + @RequestParam("reportId") String reportId, @RequestParam("resourceIndex") int resourceIndex, + @RequestParam("threadNum") int threadNum) { + byte[] bytes = jmeterFileService.downloadZip(testId, resourceId, ratio, startTime, reportId, resourceIndex, threadNum); + return ResponseEntity.ok() + .contentType(MediaType.parseMediaType("application/octet-stream")) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + testId + ".zip\"") + .body(bytes); + } + +} diff --git a/backend/src/main/java/io/metersphere/performance/engine/EngineContext.java b/backend/src/main/java/io/metersphere/performance/engine/EngineContext.java index ab640de3af..ec643c499a 100644 --- a/backend/src/main/java/io/metersphere/performance/engine/EngineContext.java +++ b/backend/src/main/java/io/metersphere/performance/engine/EngineContext.java @@ -15,7 +15,6 @@ public class EngineContext { private Integer resourceIndex; private Map properties = new HashMap<>(); private Map testData = new HashMap<>(); - private Map env = new HashMap<>(); private Map testJars = new HashMap<>(); public String getTestId() { @@ -50,14 +49,6 @@ public class EngineContext { this.properties.putAll(props); } - public Map getEnv() { - return env; - } - - public void setEnv(Map env) { - this.env = env; - } - public Object getProperty(String key) { return this.properties.get(key); } diff --git a/backend/src/main/java/io/metersphere/performance/engine/EngineFactory.java b/backend/src/main/java/io/metersphere/performance/engine/EngineFactory.java index 7aaefeb9b7..0e1520f2fc 100644 --- a/backend/src/main/java/io/metersphere/performance/engine/EngineFactory.java +++ b/backend/src/main/java/io/metersphere/performance/engine/EngineFactory.java @@ -11,7 +11,6 @@ import io.metersphere.commons.constants.FileType; import io.metersphere.commons.constants.ResourcePoolTypeEnum; import io.metersphere.commons.exception.MSException; import io.metersphere.commons.utils.LogUtil; -import io.metersphere.config.KafkaProperties; import io.metersphere.i18n.Translator; import io.metersphere.performance.engine.docker.DockerTestEngine; import io.metersphere.performance.parse.EngineSourceParser; @@ -34,7 +33,6 @@ import java.util.stream.Collectors; public class EngineFactory { private static FileService fileService; private static TestResourcePoolService testResourcePoolService; - private static KafkaProperties kafkaProperties; private static Class kubernetesTestEngineClass; static { @@ -99,13 +97,6 @@ public class EngineFactory { engineContext.setStartTime(startTime); engineContext.setReportId(reportId); engineContext.setResourceIndex(resourceIndex); - HashMap env = new HashMap() {{ - put("BOOTSTRAP_SERVERS", kafkaProperties.getBootstrapServers()); - put("LOG_TOPIC", kafkaProperties.getLog().getTopic()); - put("REPORT_ID", reportId); - put("RESOURCE_ID", resourceId); - }}; - engineContext.setEnv(env); if (StringUtils.isNotEmpty(loadTest.getLoadConfiguration())) { final JSONArray jsonArray = JSONObject.parseArray(loadTest.getLoadConfiguration()); @@ -197,9 +188,4 @@ public class EngineFactory { public void setTestResourcePoolService(TestResourcePoolService testResourcePoolService) { EngineFactory.testResourcePoolService = testResourcePoolService; } - - @Resource - public void setKafkaProperties(KafkaProperties kafkaProperties) { - EngineFactory.kafkaProperties = kafkaProperties; - } } diff --git a/backend/src/main/java/io/metersphere/performance/engine/docker/DockerTestEngine.java b/backend/src/main/java/io/metersphere/performance/engine/docker/DockerTestEngine.java index 3afd797fd7..360a052749 100644 --- a/backend/src/main/java/io/metersphere/performance/engine/docker/DockerTestEngine.java +++ b/backend/src/main/java/io/metersphere/performance/engine/docker/DockerTestEngine.java @@ -6,17 +6,19 @@ import io.metersphere.base.domain.TestResource; import io.metersphere.commons.constants.ResourceStatusEnum; import io.metersphere.commons.exception.MSException; import io.metersphere.commons.utils.CommonBeanFactory; -import io.metersphere.commons.utils.LogUtil; +import io.metersphere.config.KafkaProperties; import io.metersphere.controller.ResultHolder; +import io.metersphere.dto.BaseSystemConfigDTO; import io.metersphere.dto.NodeDTO; import io.metersphere.i18n.Translator; import io.metersphere.performance.engine.AbstractEngine; -import io.metersphere.performance.engine.EngineContext; -import io.metersphere.performance.engine.EngineFactory; -import io.metersphere.performance.engine.docker.request.TestRequest; +import io.metersphere.performance.engine.request.StartTestRequest; +import io.metersphere.service.SystemParameterService; import org.springframework.web.client.RestTemplate; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class DockerTestEngine extends AbstractEngine { @@ -60,38 +62,38 @@ public class DockerTestEngine extends AbstractEngine { } private void runTest(TestResource resource, double ratio, int resourceIndex) { - EngineContext context = null; - try { - context = EngineFactory.createContext(loadTest, resource.getId(), ratio, this.getStartTime(), this.getReportId(), resourceIndex); - } catch (MSException e) { - LogUtil.error(e.getMessage(), e); - throw e; - } catch (Exception e) { - LogUtil.error(e.getMessage(), e); - MSException.throwException(e); - } String configuration = resource.getConfiguration(); NodeDTO node = JSON.parseObject(configuration, NodeDTO.class); String nodeIp = node.getIp(); Integer port = node.getPort(); - String testId = context.getTestId(); - String content = context.getContent(); + + BaseSystemConfigDTO baseInfo = CommonBeanFactory.getBean(SystemParameterService.class).getBaseInfo(); + KafkaProperties kafkaProperties = CommonBeanFactory.getBean(KafkaProperties.class); + String metersphereUrl = "http://localhost:8081"; + if (baseInfo != null) { + metersphereUrl = baseInfo.getUrl(); + } + + Map env = new HashMap<>(); + env.put("RATIO", "" + ratio); + env.put("RESOURCE_INDEX", "" + resourceIndex); + env.put("METERSPHERE_URL", metersphereUrl); + env.put("START_TIME", "" + this.getStartTime()); + env.put("TEST_ID", this.loadTest.getId()); + env.put("REPORT_ID", this.getReportId()); + env.put("BOOTSTRAP_SERVERS", kafkaProperties.getBootstrapServers()); + env.put("LOG_TOPIC", kafkaProperties.getLog().getTopic()); + env.put("RESOURCE_ID", resource.getId()); + env.put("THREAD_NUM", "" + threadNum); + + + StartTestRequest startTestRequest = new StartTestRequest(); + startTestRequest.setImage(JMETER_IMAGE); + startTestRequest.setEnv(env); String uri = String.format(BASE_URL + "/jmeter/container/start", nodeIp, port); - - TestRequest testRequest = new TestRequest(); - testRequest.setSize(1); - testRequest.setTestId(testId); - testRequest.setReportId(getReportId()); - testRequest.setFileString(content); - testRequest.setImage(JMETER_IMAGE); - testRequest.setTestData(context.getTestData()); - testRequest.setTestJars(context.getTestJars()); - testRequest.setEnv(context.getEnv()); - - - ResultHolder result = restTemplate.postForObject(uri, testRequest, ResultHolder.class); + ResultHolder result = restTemplate.postForObject(uri, startTestRequest, ResultHolder.class); if (result == null) { MSException.throwException(Translator.get("start_engine_fail")); } diff --git a/backend/src/main/java/io/metersphere/performance/engine/docker/request/BaseRequest.java b/backend/src/main/java/io/metersphere/performance/engine/docker/request/BaseRequest.java deleted file mode 100644 index 3d1546ba02..0000000000 --- a/backend/src/main/java/io/metersphere/performance/engine/docker/request/BaseRequest.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.metersphere.performance.engine.docker.request; - -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class BaseRequest { - private String testId; - private String reportId; -} diff --git a/backend/src/main/java/io/metersphere/performance/engine/docker/request/TestRequest.java b/backend/src/main/java/io/metersphere/performance/engine/docker/request/TestRequest.java deleted file mode 100644 index 0b548a6e4f..0000000000 --- a/backend/src/main/java/io/metersphere/performance/engine/docker/request/TestRequest.java +++ /dev/null @@ -1,18 +0,0 @@ -package io.metersphere.performance.engine.docker.request; - -import lombok.Getter; -import lombok.Setter; - -import java.util.HashMap; -import java.util.Map; - -@Getter -@Setter -public class TestRequest extends BaseRequest { - private int size; - private String fileString; - private String image; - private Map testData = new HashMap<>(); - private Map env = new HashMap<>(); - private Map testJars = new HashMap<>(); -} diff --git a/backend/src/main/java/io/metersphere/performance/engine/request/BaseRequest.java b/backend/src/main/java/io/metersphere/performance/engine/request/BaseRequest.java new file mode 100644 index 0000000000..8f0ff04aea --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/engine/request/BaseRequest.java @@ -0,0 +1,9 @@ +package io.metersphere.performance.engine.request; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class BaseRequest { +} diff --git a/backend/src/main/java/io/metersphere/performance/engine/request/StartTestRequest.java b/backend/src/main/java/io/metersphere/performance/engine/request/StartTestRequest.java new file mode 100644 index 0000000000..21fb9da7a2 --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/engine/request/StartTestRequest.java @@ -0,0 +1,14 @@ +package io.metersphere.performance.engine.request; + +import lombok.Getter; +import lombok.Setter; + +import java.util.HashMap; +import java.util.Map; + +@Getter +@Setter +public class StartTestRequest { + private String image; + private Map env = new HashMap<>(); +} diff --git a/backend/src/main/java/io/metersphere/performance/engine/request/StopTestRequest.java b/backend/src/main/java/io/metersphere/performance/engine/request/StopTestRequest.java new file mode 100644 index 0000000000..945476d943 --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/engine/request/StopTestRequest.java @@ -0,0 +1,9 @@ +package io.metersphere.performance.engine.request; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class StopTestRequest { +} diff --git a/backend/src/main/java/io/metersphere/performance/service/JmeterFileService.java b/backend/src/main/java/io/metersphere/performance/service/JmeterFileService.java new file mode 100644 index 0000000000..29dad1b809 --- /dev/null +++ b/backend/src/main/java/io/metersphere/performance/service/JmeterFileService.java @@ -0,0 +1,121 @@ +package io.metersphere.performance.service; + + +import com.alibaba.excel.util.CollectionUtils; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import io.metersphere.base.domain.LoadTestWithBLOBs; +import io.metersphere.base.mapper.LoadTestMapper; +import io.metersphere.commons.exception.MSException; +import io.metersphere.commons.utils.LogUtil; +import io.metersphere.performance.engine.EngineContext; +import io.metersphere.performance.engine.EngineFactory; +import org.apache.commons.lang3.SerializationUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +@Service +public class JmeterFileService { + + @Resource + private LoadTestMapper loadTestMapper; + + public byte[] downloadZip(String testId, String resourceId, double ratio, long startTime, String reportId, int resourceIndex, int threadNum) { + try { + LoadTestWithBLOBs loadTest = loadTestMapper.selectByPrimaryKey(testId); + // deep copy + LoadTestWithBLOBs subTest = SerializationUtils.clone(loadTest); + setThreadNum(subTest, threadNum); + EngineContext context = EngineFactory.createContext(subTest, resourceId, ratio, startTime, reportId, resourceIndex); + return zipFilesToByteArray(context); + } catch (MSException e) { + LogUtil.error(e.getMessage(), e); + throw e; + } catch (Exception e) { + LogUtil.error(e.getMessage(), e); + MSException.throwException(e); + } + return null; + } + + private void setThreadNum(LoadTestWithBLOBs t, Integer limit) { + String loadConfiguration = t.getLoadConfiguration(); + JSONArray jsonArray = JSON.parseArray(loadConfiguration); + for (int i = 0; i < jsonArray.size(); i++) { + if (jsonArray.get(i) instanceof Map) { + JSONObject o = jsonArray.getJSONObject(i); + if (StringUtils.equals(o.getString("key"), "TargetLevel")) { + o.put("value", limit); + break; + } + } + if (jsonArray.get(i) instanceof List) { + JSONArray o = jsonArray.getJSONArray(i); + for (int j = 0; j < o.size(); j++) { + JSONObject b = o.getJSONObject(j); + if (StringUtils.equals(b.getString("key"), "TargetLevel")) { + b.put("value", limit); + break; + } + } + } + } + // 设置线程数 + t.setLoadConfiguration(jsonArray.toJSONString()); + } + + private byte[] zipFilesToByteArray(EngineContext context) throws IOException { + String testId = context.getTestId(); + String fileName = testId + ".jmx"; + + Map files = new HashMap<>(); + + // 每个测试生成一个文件夹 + files.put(fileName, context.getContent().getBytes(StandardCharsets.UTF_8)); + // 保存测试数据文件 + Map testData = context.getTestData(); + if (!CollectionUtils.isEmpty(testData)) { + for (String k : testData.keySet()) { + String v = testData.get(k); + files.put("k", v.getBytes(StandardCharsets.UTF_8)); + } + } + + // 保存 byte[] jar + Map jarFiles = context.getTestJars(); + if (!CollectionUtils.isEmpty(jarFiles)) { + for (String k : jarFiles.keySet()) { + byte[] v = jarFiles.get(k); + files.put("k", v); + } + } + + return listBytesToZip(files); + } + + + private byte[] listBytesToZip(Map mapReport) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ZipOutputStream zos = new ZipOutputStream(baos); + for (Map.Entry report : mapReport.entrySet()) { + ZipEntry entry = new ZipEntry(report.getKey()); + entry.setSize(report.getValue().length); + zos.putNextEntry(entry); + zos.write(report.getValue()); + } + zos.closeEntry(); + zos.close(); + return baos.toByteArray(); + } +} diff --git a/backend/src/main/java/io/metersphere/xpack b/backend/src/main/java/io/metersphere/xpack index b5643ac73f..c04791219c 160000 --- a/backend/src/main/java/io/metersphere/xpack +++ b/backend/src/main/java/io/metersphere/xpack @@ -1 +1 @@ -Subproject commit b5643ac73fb4a2db3f11b23a81848990d8e78703 +Subproject commit c04791219ce92fd77c7f5f18a3f625f2083194b0