test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(接口测试): 并发任务分配增加 加权轮询算法 动态计算权重分配节点

This commit is contained in:
fit2-zhao 2022-04-15 10:44:30 +08:00 committed by 刘瑞斌
parent 0c82fb479d
commit 88e7c7e545
12 changed files with 281 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backend/src/main/java/io/metersphere/api/exec/api/ApiCaseParallelExecuteService.java
View File

@ -4,19 +4,18 @@ import io.metersphere.api.exec.queue.DBTestQueue;
import io.metersphere.api.exec.scenario.ApiScenarioSerialService;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.api.jmeter.utils.SmoothWeighted;
import io.metersphere.base.domain.ApiDefinitionExecResult;
import io.metersphere.base.domain.TestResource;
import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.dto.RunModeConfigDTO;
import io.metersphere.vo.BooleanPool;
import org.apache.commons.collections4.MapUtils;
import org.apache.jorphan.collections.HashTree;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Service
@ -25,19 +24,20 @@ public class ApiCaseParallelExecuteService {
private ApiScenarioSerialService apiScenarioSerialService;
@Resource
private JMeterService jMeterService;
@Resource
private RedisTemplate<String, Object> redisTemplate;
public void parallel(Map<String, ApiDefinitionExecResult> executeQueue, RunModeConfigDTO config, DBTestQueue executionQueue, String runMode) {
List<TestResource> resources = new ArrayList<>();
BooleanPool pool = GenerateHashTreeUtil.isResourcePool(config.getResourcePoolId());
// 初始化分配策略
if (pool.isPool()) {
resources = GenerateHashTreeUtil.setPoolResource(config.getResourcePoolId());
SmoothWeighted.setServerConfig(config.getResourcePoolId(), redisTemplate);
}
for (String testId : executeQueue.keySet()) {
ApiDefinitionExecResult result = executeQueue.get(testId);
String reportId = result.getId();
JmeterRunRequestDTO runRequest = new JmeterRunRequestDTO(testId, reportId, runMode, null);
runRequest.setPool(GenerateHashTreeUtil.isResourcePool(config.getResourcePoolId()));
runRequest.setPool(pool);
runRequest.setTestPlanReportId(executionQueue.getReportId());
runRequest.setPoolId(config.getResourcePoolId());
runRequest.setReportType(executionQueue.getReportType());
@ -50,7 +50,7 @@ public class ApiCaseParallelExecuteService {
HashTree hashTree = apiScenarioSerialService.generateHashTree(testId, config.getEnvMap(), runRequest);
runRequest.setHashTree(hashTree);
}
jMeterService.run(runRequest, resources);
jMeterService.run(runRequest);
}
}
}

4
backend/src/main/java/io/metersphere/api/exec/api/ApiExecuteService.java
View File

@ -119,7 +119,7 @@ public class ApiExecuteService {
}
// 调用执行方法
JmeterRunRequestDTO runRequest = new JmeterRunRequestDTO(testCaseWithBLOBs.getId(), StringUtils.isEmpty(request.getReportId()) ? request.getId() : request.getReportId(), request.getRunMode(), jmeterHashTree);
jMeterService.run(runRequest, new ArrayList<>());
jMeterService.run(runRequest);
} catch (Exception ex) {
ApiDefinitionExecResult result = apiDefinitionExecResultMapper.selectByPrimaryKey(request.getReportId());
if (result != null) {
@ -188,7 +188,7 @@ public class ApiExecuteService {
this.put("SYN_RES", request.isSyncResult());
}});
// 开始执行
jMeterService.run(runRequest, new ArrayList<>());
jMeterService.run(runRequest);
return new MsExecResponseDTO(runRequest.getTestId(), runRequest.getReportId(), runMode);
}

2
backend/src/main/java/io/metersphere/api/exec/queue/ExecThreadPoolExecutor.java
View File

@ -105,11 +105,11 @@ public class ExecThreadPoolExecutor {
public void setCorePoolSize(int maximumPoolSize) {
try {
if (maximumPoolSize != threadPool.getMaximumPoolSize()) {
threadPool.setMaximumPoolSize(maximumPoolSize);
int corePoolSize = maximumPoolSize > 500 ? 500 : maximumPoolSize;
if (corePoolSize > CORE_POOL_SIZE) {
threadPool.setCorePoolSize(corePoolSize);
}
threadPool.setMaximumPoolSize(maximumPoolSize);
threadPool.allowCoreThreadTimeOut(true);
LoggerUtil.info("AllCoreThreads: " + threadPool.prestartAllCoreThreads());
}

2
backend/src/main/java/io/metersphere/api/exec/scenario/ApiScenarioExecuteService.java
View File

@ -369,7 +369,7 @@ public class ApiScenarioExecuteService {
// 调用执行方法
JmeterRunRequestDTO runRequest = new JmeterRunRequestDTO(request.getId(), request.getId(), runMode, hashTree);
runRequest.setDebug(true);
jMeterService.run(runRequest, new ArrayList<>());
jMeterService.run(runRequest);
return request.getId();
}

15
backend/src/main/java/io/metersphere/api/exec/scenario/ApiScenarioParallelService.java
View File

@ -6,29 +6,30 @@ import io.metersphere.api.dto.automation.RunScenarioRequest;
import io.metersphere.api.exec.queue.DBTestQueue;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.base.domain.TestResource;
import io.metersphere.api.jmeter.utils.SmoothWeighted;
import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.utils.LoggerUtil;
import io.metersphere.vo.BooleanPool;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Service
public class ApiScenarioParallelService {
@Resource
private JMeterService jMeterService;
@Resource
private RedisTemplate<String, Object> redisTemplate;
public void parallel(Map<String, RunModeDataDTO> executeQueue, RunScenarioRequest request, String serialReportId, DBTestQueue executionQueue) {
List<TestResource> resources = new ArrayList<>();
// 初始化分配策略
BooleanPool pool = GenerateHashTreeUtil.isResourcePool(request.getConfig().getResourcePoolId());
if (pool.isPool()) {
resources = GenerateHashTreeUtil.setPoolResource(request.getConfig().getResourcePoolId());
SmoothWeighted.setServerConfig(request.getConfig().getResourcePoolId(), redisTemplate);
}
for (String reportId : executeQueue.keySet()) {
RunModeDataDTO dataDTO = executeQueue.get(reportId);
@ -46,10 +47,10 @@ public class ApiScenarioParallelService {
LoggerUtil.debug("Scenario run-开始并发执行：" + JSON.toJSONString(request));
}
// 本地执行生成hashTree
if (request.getConfig() != null && !runRequest.getPool().isPool()) {
if (!pool.isPool()) {
runRequest.setHashTree(GenerateHashTreeUtil.generateHashTree(dataDTO.getScenario(), dataDTO.getPlanEnvMap(), runRequest));
}
jMeterService.run(runRequest, resources);
jMeterService.run(runRequest);
}
}
}

20
backend/src/main/java/io/metersphere/api/exec/scenario/ApiScenarioSerialService.java
View File

@ -15,6 +15,7 @@ import io.metersphere.api.dto.definition.request.sampler.MsJDBCSampler;
import io.metersphere.api.dto.definition.request.sampler.MsTCPSampler;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.api.jmeter.utils.SmoothWeighted;
import io.metersphere.api.service.ApiExecutionQueueService;
import io.metersphere.api.service.ApiTestEnvironmentService;
import io.metersphere.api.service.RemakeReportService;
@ -33,10 +34,13 @@ import io.metersphere.plugin.core.MsTestElement;
import io.metersphere.utils.LoggerUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.jorphan.collections.HashTree;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.*;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.Map;
@Service
public class ApiScenarioSerialService {
@ -58,6 +62,8 @@ public class ApiScenarioSerialService {
private ApiScenarioEnvService apiScenarioEnvService;
@Resource
private ObjectMapper mapper;
@Resource
private RedisTemplate<String, Object> redisTemplate;
public void serial(ApiExecutionQueue executionQueue, ApiExecutionQueueDetail queue) {
LoggerUtil.debug("Scenario run-执行脚本装载-进入串行准备");
@ -126,13 +132,11 @@ public class ApiScenarioSerialService {
if (queue != null) {
runRequest.setPlatformUrl(queue.getId());
}
List<TestResource> resources = new ArrayList<>();
if (runRequest.getPool().isPool()) {
resources = GenerateHashTreeUtil.setPoolResource(runRequest.getPoolId());
SmoothWeighted.setServerConfig(runRequest.getPoolId(), redisTemplate);
}
// 开始执行
jMeterService.run(runRequest, resources);
jMeterService.run(runRequest);
} catch (Exception e) {
RemakeReportService remakeReportService = CommonBeanFactory.getBean(RemakeReportService.class);
remakeReportService.remake(runRequest);
@ -206,7 +210,7 @@ public class ApiScenarioSerialService {
list.addAll(elements);
}
if (element.getString("type").equals("HTTPSamplerProxy")) {
MsHTTPSamplerProxy httpSamplerProxy = JSON.parseObject(api, MsHTTPSamplerProxy.class,Feature.DisableSpecialKeyDetect);
MsHTTPSamplerProxy httpSamplerProxy = JSON.parseObject(api, MsHTTPSamplerProxy.class, Feature.DisableSpecialKeyDetect);
httpSamplerProxy.setHashTree(list);
httpSamplerProxy.setName(planId);
if (StringUtils.isNotEmpty(envId)) {
@ -215,7 +219,7 @@ public class ApiScenarioSerialService {
return httpSamplerProxy;
}
if (element.getString("type").equals("TCPSampler")) {
MsTCPSampler msTCPSampler = JSON.parseObject(api, MsTCPSampler.class,Feature.DisableSpecialKeyDetect);
MsTCPSampler msTCPSampler = JSON.parseObject(api, MsTCPSampler.class, Feature.DisableSpecialKeyDetect);
if (StringUtils.isNotEmpty(envId)) {
msTCPSampler.setUseEnvironment(envId);
}
@ -224,7 +228,7 @@ public class ApiScenarioSerialService {
return msTCPSampler;
}
if (element.getString("type").equals("DubboSampler")) {
MsDubboSampler dubboSampler = JSON.parseObject(api, MsDubboSampler.class,Feature.DisableSpecialKeyDetect);
MsDubboSampler dubboSampler = JSON.parseObject(api, MsDubboSampler.class, Feature.DisableSpecialKeyDetect);
if (StringUtils.isNotEmpty(envId)) {
dubboSampler.setUseEnvironment(envId);
}

4
backend/src/main/java/io/metersphere/api/exec/utils/GenerateHashTreeUtil.java
View File

@ -14,7 +14,6 @@ import io.metersphere.api.jmeter.ResourcePoolCalculation;
import io.metersphere.api.service.ApiExecutionQueueService;
import io.metersphere.api.service.RemakeReportService;
import io.metersphere.base.domain.ApiScenarioWithBLOBs;
import io.metersphere.base.domain.TestResource;
import io.metersphere.base.domain.TestResourcePool;
import io.metersphere.base.mapper.TestResourcePoolMapper;
import io.metersphere.commons.constants.ResourcePoolTypeEnum;
@ -23,6 +22,7 @@ import io.metersphere.commons.utils.CommonBeanFactory;
import io.metersphere.commons.utils.LogUtil;
import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.dto.JvmInfoDTO;
import io.metersphere.dto.ResultDTO;
import io.metersphere.dto.RunModeConfigDTO;
import io.metersphere.plugin.core.MsTestElement;
@ -98,7 +98,7 @@ public class GenerateHashTreeUtil {
return pool;
}
public static List<TestResource> setPoolResource(String id) {
public static List<JvmInfoDTO> setPoolResource(String id) {
if (GenerateHashTreeUtil.isResourcePool(id).isPool() && !GenerateHashTreeUtil.isResourcePool(id).isK8s()) {
ResourcePoolCalculation resourcePoolCalculation = CommonBeanFactory.getBean(ResourcePoolCalculation.class);
return resourcePoolCalculation.getPools(id);

59
backend/src/main/java/io/metersphere/api/jmeter/JMeterService.java
View File

@ -3,6 +3,8 @@ package io.metersphere.api.jmeter;
import com.alibaba.fastjson.JSON;
import io.metersphere.api.exec.queue.ExecThreadPoolExecutor;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.api.jmeter.utils.ServerConfig;
import io.metersphere.api.jmeter.utils.SmoothWeighted;
import io.metersphere.api.service.ApiScenarioReportService;
import io.metersphere.api.service.RemakeReportService;
import io.metersphere.base.domain.TestResource;
@ -14,6 +16,7 @@ import io.metersphere.config.KafkaConfig;
import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.BaseSystemConfigDTO;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.dto.JvmInfoDTO;
import io.metersphere.dto.NodeDTO;
import io.metersphere.jmeter.JMeterBase;
import io.metersphere.jmeter.LocalRunner;
@ -29,6 +32,7 @@ import org.apache.jmeter.testelement.TestElement;
import org.apache.jmeter.util.JMeterUtils;
import org.apache.jorphan.collections.HashTree;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
@ -45,6 +49,8 @@ public class JMeterService {
private JmeterProperties jmeterProperties;
@Resource
private RestTemplate restTemplate;
@Resource
private RedisTemplate<String, Object> redisTemplate;
@PostConstruct
private void init() {
@ -114,7 +120,7 @@ public class JMeterService {
runner.run(request.getReportId());
}
private void runNode(JmeterRunRequestDTO request, List<TestResource> resources) {
private void runNode(JmeterRunRequestDTO request) {
// 获取可以执行的资源池
BaseSystemConfigDTO baseInfo = CommonBeanFactory.getBean(SystemParameterService.class).getBaseInfo();
// 占位符
@ -147,38 +153,34 @@ public class JMeterService {
MSException.throwException(e.getMessage());
}
} else {
this.send(request, resources);
this.send(request);
}
}
private void send(JmeterRunRequestDTO request, List<TestResource> resources) {
private void send(JmeterRunRequestDTO request) {
try {
if (StringUtils.isNotEmpty(request.getPoolId()) && CollectionUtils.isEmpty(resources)) {
resources = GenerateHashTreeUtil.setPoolResource(request.getPoolId());
if (redisTemplate.opsForValue().get(SmoothWeighted.EXEC_INDEX + request.getPoolId()) != null) {
long index = Long.parseLong(redisTemplate.opsForValue().get(SmoothWeighted.EXEC_INDEX + request.getPoolId()).toString());
redisTemplate.opsForValue().set(SmoothWeighted.EXEC_INDEX + request.getPoolId(), index + 1);
}
if (CollectionUtils.isEmpty(resources)) {
ServerConfig config = SmoothWeighted.calculate(request.getPoolId(), redisTemplate);
if (config == null) {
config = SmoothWeighted.getResource(request.getPoolId());
}
if (config == null) {
LoggerUtil.info("未获取到资源池，请检查配置【系统设置-系统-测试资源池】");
RemakeReportService remakeReportService = CommonBeanFactory.getBean(RemakeReportService.class);
remakeReportService.remake(request);
return;
}
int index = (int) (Math.random() * resources.size());
TestResource testResource = resources.get(index);
String configuration = testResource.getConfiguration();
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
request.setCorePoolSize(node.getMaxConcurrency());
request.setEnable(node.isEnable());
String nodeIp = node.getIp();
Integer port = node.getPort();
String uri = String.format(BASE_URL + "/jmeter/api/start", nodeIp, port);
LoggerUtil.info("开始发送请求【 " + request.getReportId() + " 】,资源【 " + request.getTestId() + " 】" + uri + " 节点执行");
ResponseEntity<String> result = restTemplate.postForEntity(uri, request, String.class);
request.setCorePoolSize(config.getCorePoolSize());
request.setEnable(config.isEnable());
LoggerUtil.info("开始发送请求【 " + request.getReportId() + " 】,资源【 " + request.getTestId() + " 】" + config.getUrl() + " 节点执行");
ResponseEntity<String> result = restTemplate.postForEntity(config.getUrl(), request, String.class);
if (result == null || !StringUtils.equals("SUCCESS", result.getBody())) {
RemakeReportService remakeReportService = CommonBeanFactory.getBean(RemakeReportService.class);
remakeReportService.remake(request);
LoggerUtil.error("发送请求[ " + request.getTestId() + " ] 到" + uri + " 节点执行失败");
LoggerUtil.error("发送请求[ " + request.getTestId() + " ] 到" + config.getUrl() + " 节点执行失败");
LoggerUtil.info(result.getBody());
}
} catch (Exception e) {
@ -190,9 +192,18 @@ public class JMeterService {
}
public void run(JmeterRunRequestDTO request) {
if (request.getPool().isPool()) {
this.runNode(request);
} else {
CommonBeanFactory.getBean(ExecThreadPoolExecutor.class).addTask(request);
}
}
@Deprecated
public void run(JmeterRunRequestDTO request, List<TestResource> resources) {
if (request.getPool().isPool()) {
this.runNode(request, resources);
this.runNode(request);
} else {
CommonBeanFactory.getBean(ExecThreadPoolExecutor.class).addTask(request);
}
@ -204,13 +215,13 @@ public class JMeterService {
public boolean getRunningQueue(String poolId, String reportId) {
try {
List<TestResource> resources = GenerateHashTreeUtil.setPoolResource(poolId);
List<JvmInfoDTO> resources = GenerateHashTreeUtil.setPoolResource(poolId);
if (CollectionUtils.isEmpty(resources)) {
return false;
}
boolean isRunning = false;
for (TestResource testResource : resources) {
String configuration = testResource.getConfiguration();
for (JvmInfoDTO testResource : resources) {
String configuration = testResource.getTestResource().getConfiguration();
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
String nodeIp = node.getIp();
Integer port = node.getPort();

23
backend/src/main/java/io/metersphere/api/jmeter/ResourcePoolCalculation.java
View File

@ -7,9 +7,11 @@ import io.metersphere.base.domain.TestResourcePool;
import io.metersphere.base.domain.TestResourcePoolExample;
import io.metersphere.base.mapper.TestResourceMapper;
import io.metersphere.base.mapper.TestResourcePoolMapper;
import io.metersphere.commons.utils.BeanUtils;
import io.metersphere.dto.JvmInfoDTO;
import io.metersphere.dto.NodeDTO;
import io.metersphere.dto.TestResourceDTO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.RestTemplate;
@ -31,22 +33,22 @@ public class ResourcePoolCalculation {
private static final String BASE_URL = "http://%s:%d";
private String getNodeJvmInfo(String uri) {
private JvmInfoDTO getNodeJvmInfo(String uri) {
try {
return restTemplate.getForObject(uri, String.class);
return restTemplate.getForObject(uri, JvmInfoDTO.class);
} catch (Exception e) {
return null;
}
}
public List<TestResource> getPools(String resourcePoolId) {
public List<JvmInfoDTO> getPools(String resourcePoolId) {
// 获取可以执行的资源池
TestResourcePoolExample example = new TestResourcePoolExample();
example.createCriteria().andStatusEqualTo("VALID").andTypeEqualTo("NODE").andIdEqualTo(resourcePoolId);
List<TestResourcePool> pools = testResourcePoolMapper.selectByExample(example);
// 按照NODE节点的可用内存空间大小排序
List<TestResource> availableNodes = new ArrayList<>();
List<JvmInfoDTO> availableNodes = new ArrayList<>();
if (CollectionUtils.isNotEmpty(pools)) {
List<String> poolIds = pools.stream().map(pool -> pool.getId()).collect(Collectors.toList());
TestResourceExample resourceExample = new TestResourceExample();
@ -57,12 +59,15 @@ public class ResourcePoolCalculation {
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
String nodeIp = node.getIp();
Integer port = node.getPort();
String uri = String.format(BASE_URL + "/jmeter/status", nodeIp, port);
String status = this.getNodeJvmInfo(uri);
if (!StringUtils.equals(status, "OK")) {
String uri = String.format(BASE_URL + "/jmeter/getJvmInfo", nodeIp, port);
JvmInfoDTO nodeJvm = this.getNodeJvmInfo(uri);
if (nodeJvm == null) {
continue;
}
availableNodes.add(testResource);
TestResourceDTO dto = new TestResourceDTO();
BeanUtils.copyBean(dto, testResource);
nodeJvm.setTestResource(dto);
availableNodes.add(nodeJvm);
}
}
return availableNodes;

68
backend/src/main/java/io/metersphere/api/jmeter/utils/ServerConfig.java Normal file
View File

@ -0,0 +1,68 @@
package io.metersphere.api.jmeter.utils;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import java.io.Serializable;
@Data
public class ServerConfig implements Serializable {
//服务名称
public String url;
//初始权重
public int weight;
//当前权重
public int currentWeight;
// 资源池并发数
private int corePoolSize;
// 是否开启JAR同步
private boolean enable;
public ServerConfig() {
}
public ServerConfig(String url, int weight) {
this.url = url;
this.weight = weight;
}
public ServerConfig(String url, int weight, int currentWeight) {
this.url = url;
this.weight = weight;
this.currentWeight = currentWeight;
}
public ServerConfig(String url, int weight, int currentWeight, int corePoolSize, boolean enable) {
this.url = url;
this.weight = weight;
this.currentWeight = currentWeight;
this.corePoolSize = corePoolSize;
this.enable = enable;
}
@Override
public String toString() {
return "ServerConfig{" + "url='" + url + '}';
}
@Override
public boolean equals(Object value) {
if (value != null && value instanceof ServerConfig) {
return StringUtils.equals(((ServerConfig) value).getUrl(), this.getUrl());
}
return false;
}
@Override
public ServerConfig clone() throws CloneNotSupportedException {
ServerConfig serverConfig = new ServerConfig();
serverConfig.setCurrentWeight(this.currentWeight);
serverConfig.setUrl(this.url);
serverConfig.setWeight(this.weight);
return serverConfig;
}
}

130
backend/src/main/java/io/metersphere/api/jmeter/utils/SmoothWeighted.java Normal file
View File

@ -0,0 +1,130 @@
package io.metersphere.api.jmeter.utils;
import com.alibaba.fastjson.JSON;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.dto.JvmInfoDTO;
import io.metersphere.dto.NodeDTO;
import io.metersphere.dto.TestResourceDTO;
import io.metersphere.utils.LoggerUtil;
import io.metersphere.vo.BooleanPool;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.redis.core.RedisTemplate;
import java.util.*;
/**
* 平滑加权轮询算法
*/
public class SmoothWeighted {
private static final String BASE_URL = "http://%s:%d";
public static final String CONFIG = "CONFIG_";
public static final String EXEC_INDEX = "EXEC_INDEX_";
public static void setServerConfig(String poolId, RedisTemplate client) {
if (StringUtils.isNotEmpty(poolId)) {
return;
}
List<JvmInfoDTO> resources = new ArrayList<>();
BooleanPool pool = GenerateHashTreeUtil.isResourcePool(poolId);
if (pool.isPool()) {
resources = GenerateHashTreeUtil.setPoolResource(poolId);
}
if (CollectionUtils.isEmpty(resources)) {
client.delete(CONFIG + poolId);
client.delete(EXEC_INDEX + poolId);
return;
}
Map<String, ServerConfig> configs = new HashMap<>();
for (JvmInfoDTO jvmInfoDTO : resources) {
String configuration = jvmInfoDTO.getTestResource().getConfiguration();
if (StringUtils.isNotEmpty(configuration)) {
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
String uri = String.format(BASE_URL + "/jmeter/api/start", node.getIp(), node.getPort());
configs.put(uri, new ServerConfig(uri, 1, 1, node.getMaxConcurrency(), node.isEnable()));
}
}
if (client.opsForValue().get(CONFIG + poolId) == null) {
client.opsForValue().set(CONFIG + poolId, new ArrayList<>(configs.values()));
} else {
// 合并相同节点信息，更改权重
List<ServerConfig> serverConfigs = (List<ServerConfig>) client.opsForValue().get(CONFIG + poolId);
serverConfigs.forEach(item -> {
if (configs.containsKey(item.getUrl())) {
item.setCorePoolSize(configs.get(item.getUrl()).getCorePoolSize());
item.setEnable(configs.get(item.getUrl()).isEnable());
item.setWeight(configs.get(item.getUrl()).getWeight());
}
});
// 添加新增节点
configs.forEach((k, v) -> {
if (!serverConfigs.contains(v)) {
serverConfigs.add(v);
}
});
// 异常已经废弃的节点
serverConfigs.removeIf(item -> !configs.containsKey(item.getUrl()));
client.opsForValue().set(CONFIG + poolId, serverConfigs);
}
if (client.opsForValue().get(EXEC_INDEX + poolId) == null) {
client.opsForValue().set(EXEC_INDEX + poolId, 1);
}
}
public static ServerConfig calculate(String poolId, RedisTemplate client) {
if (client.opsForValue().get(EXEC_INDEX + poolId) == null) {
client.opsForValue().set(EXEC_INDEX + poolId, 1);
}
List<ServerConfig> serverList = new ArrayList<>();
if (client.opsForValue().get(CONFIG + poolId) != null) {
serverList = (List<ServerConfig>) client.opsForValue().get(CONFIG + poolId);
}
// 最大权重
int weightSum = serverList.stream().map(ServerConfig::getWeight).reduce((x, y) -> x += y).get();
long execIndex = Long.parseLong(client.opsForValue().get(EXEC_INDEX + poolId).toString());
// 选出当前有效权重最大的实例，将当前有效权重currentWeight减去所有实例的"权重和"（weightSum），且变量tmpSv指向此位置；
ServerConfig max = Collections.max(serverList, Comparator.comparingLong(ServerConfig::getCurrentWeight));
// 选中的实例
ServerConfig tmpConfig = null;
for (ServerConfig serverConfig : serverList) {
if (max.equals(serverConfig)) {
serverConfig.setCurrentWeight(serverConfig.getCurrentWeight() - weightSum);
if (tmpConfig == null || serverConfig.getCurrentWeight() > tmpConfig.getCurrentWeight()) {
tmpConfig = serverConfig;
}
}
//将每个实例的当前有效权重currentWeight都加上配置权重weight；
serverConfig.setCurrentWeight(serverConfig.getCurrentWeight() + serverConfig.getWeight());
}
// 选中前的当前权重
LoggerUtil.info("第" + (execIndex) + "次选中前的当前权重:" + serverList.toString());
if (client.opsForValue().get(CONFIG + poolId) != null) {
client.opsForValue().set(CONFIG + poolId, serverList);
}
return tmpConfig;
}
public static ServerConfig getResource(String poolId) {
BooleanPool pool = GenerateHashTreeUtil.isResourcePool(poolId);
if (pool.isPool()) {
List<JvmInfoDTO> resources = GenerateHashTreeUtil.setPoolResource(poolId);
if (CollectionUtils.isNotEmpty(resources)) {
int index = (int) (Math.random() * resources.size());
TestResourceDTO testResource = resources.get(index).getTestResource();
NodeDTO node = JSON.parseObject(testResource.getConfiguration(), NodeDTO.class);
String nodeIp = node.getIp();
Integer port = node.getPort();
String uri = String.format(BASE_URL + "/jmeter/api/start", nodeIp, port);
return new ServerConfig(uri, 1, 1, node.getMaxConcurrency(), node.isEnable());
}
}
return null;
}
}

2
backend/src/main/java/io/metersphere/api/service/ApiAutomationService.java
View File

@ -10,7 +10,6 @@ import io.metersphere.api.dto.automation.parse.ApiScenarioImportUtil;
import io.metersphere.api.dto.automation.parse.ScenarioImport;
import io.metersphere.api.dto.automation.parse.ScenarioImportParserFactory;
import io.metersphere.api.dto.datacount.ApiDataCountResult;
import io.metersphere.api.dto.datacount.ApiMethodUrlDTO;
import io.metersphere.api.dto.definition.ApiTestCaseInfo;
import io.metersphere.api.dto.definition.RunDefinitionRequest;
import io.metersphere.api.dto.definition.request.*;
@ -459,7 +458,6 @@ public class ApiAutomationService {
scenario.setUpdateTime(System.currentTimeMillis());
scenario.setDescription(request.getDescription());
scenario.setCreateUser(SessionUtils.getUserId());
scenario.setScenarioDefinition(JSON.toJSONString(request.getScenarioDefinition()));
Boolean isValidEnum = EnumUtils.isValidEnum(EnvironmentType.class, request.getEnvironmentType());
if (BooleanUtils.isTrue(isValidEnum)) {