test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(接口测试): 并发任务分配增加 加权轮询算法 动态计算权重分配节点

This commit is contained in:
fit2-zhao 2022-04-15 10:44:30 +08:00 committed by fit2-zhao
parent 87cc82ee38
commit c79dbfee90
12 changed files with 281 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backend/src/main/java/io/metersphere/api/exec/api/ApiCaseParallelExecuteService.java
View File

@ -4,19 +4,18 @@ import io.metersphere.api.exec.queue.DBTestQueue;
import io.metersphere.api.exec.scenario.ApiScenarioSerialService;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.api.jmeter.utils.SmoothWeighted;
import io.metersphere.base.domain.ApiDefinitionExecResult;
import io.metersphere.base.domain.TestResource;
import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.dto.RunModeConfigDTO;
import io.metersphere.vo.BooleanPool;
import org.apache.commons.collections4.MapUtils;
import org.apache.jorphan.collections.HashTree;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Service
@ -25,19 +24,20 @@ public class ApiCaseParallelExecuteService {
private ApiScenarioSerialService apiScenarioSerialService;
@Resource
private JMeterService jMeterService;
@Resource
private RedisTemplate<String, Object> redisTemplate;
public void parallel(Map<String, ApiDefinitionExecResult> executeQueue, RunModeConfigDTO config, DBTestQueue executionQueue, String runMode) {
List<TestResource> resources = new ArrayList<>();
BooleanPool pool = GenerateHashTreeUtil.isResourcePool(config.getResourcePoolId());
// 初始化分配策略
if (pool.isPool()) {
resources = GenerateHashTreeUtil.setPoolResource(config.getResourcePoolId());
SmoothWeighted.setServerConfig(config.getResourcePoolId(), redisTemplate);
}
for (String testId : executeQueue.keySet()) {
ApiDefinitionExecResult result = executeQueue.get(testId);
String reportId = result.getId();
JmeterRunRequestDTO runRequest = new JmeterRunRequestDTO(testId, reportId, runMode, null);
runRequest.setPool(GenerateHashTreeUtil.isResourcePool(config.getResourcePoolId()));
runRequest.setPool(pool);
runRequest.setTestPlanReportId(executionQueue.getReportId());
runRequest.setPoolId(config.getResourcePoolId());
runRequest.setReportType(executionQueue.getReportType());
@ -50,7 +50,7 @@ public class ApiCaseParallelExecuteService {
HashTree hashTree = apiScenarioSerialService.generateHashTree(testId, config.getEnvMap(), runRequest);
runRequest.setHashTree(hashTree);
}
jMeterService.run(runRequest, resources);
jMeterService.run(runRequest);
}
}
}

4
backend/src/main/java/io/metersphere/api/exec/api/ApiExecuteService.java
View File

@ -119,7 +119,7 @@ public class ApiExecuteService {
}
// 调用执行方法
JmeterRunRequestDTO runRequest = new JmeterRunRequestDTO(testCaseWithBLOBs.getId(), StringUtils.isEmpty(request.getReportId()) ? request.getId() : request.getReportId(), request.getRunMode(), jmeterHashTree);
jMeterService.run(runRequest, new ArrayList<>());
jMeterService.run(runRequest);
} catch (Exception ex) {
ApiDefinitionExecResult result = apiDefinitionExecResultMapper.selectByPrimaryKey(request.getReportId());
if (result != null) {
@ -188,7 +188,7 @@ public class ApiExecuteService {
this.put("SYN_RES", request.isSyncResult());
}});
// 开始执行
jMeterService.run(runRequest, new ArrayList<>());
jMeterService.run(runRequest);
return new MsExecResponseDTO(runRequest.getTestId(), runRequest.getReportId(), runMode);
}

2
backend/src/main/java/io/metersphere/api/exec/queue/ExecThreadPoolExecutor.java
View File

@ -105,11 +105,11 @@ public class ExecThreadPoolExecutor {
public void setCorePoolSize(int maximumPoolSize) {
try {
if (maximumPoolSize != threadPool.getMaximumPoolSize()) {
threadPool.setMaximumPoolSize(maximumPoolSize);
int corePoolSize = maximumPoolSize > 500 ? 500 : maximumPoolSize;
if (corePoolSize > CORE_POOL_SIZE) {
threadPool.setCorePoolSize(corePoolSize);
}
threadPool.setMaximumPoolSize(maximumPoolSize);
threadPool.allowCoreThreadTimeOut(true);
LoggerUtil.info("AllCoreThreads: " + threadPool.prestartAllCoreThreads());
}

2
backend/src/main/java/io/metersphere/api/exec/scenario/ApiScenarioExecuteService.java
View File

@ -369,7 +369,7 @@ public class ApiScenarioExecuteService {
// 调用执行方法
JmeterRunRequestDTO runRequest = new JmeterRunRequestDTO(request.getId(), request.getId(), runMode, hashTree);
runRequest.setDebug(true);
jMeterService.run(runRequest, new ArrayList<>());
jMeterService.run(runRequest);
return request.getId();
}

15
backend/src/main/java/io/metersphere/api/exec/scenario/ApiScenarioParallelService.java
View File

@ -6,29 +6,30 @@ import io.metersphere.api.dto.automation.RunScenarioRequest;
import io.metersphere.api.exec.queue.DBTestQueue;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.base.domain.TestResource;
import io.metersphere.api.jmeter.utils.SmoothWeighted;
import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.utils.LoggerUtil;
import io.metersphere.vo.BooleanPool;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Service
public class ApiScenarioParallelService {
@Resource
private JMeterService jMeterService;
@Resource
private RedisTemplate<String, Object> redisTemplate;
public void parallel(Map<String, RunModeDataDTO> executeQueue, RunScenarioRequest request, String serialReportId, DBTestQueue executionQueue) {
List<TestResource> resources = new ArrayList<>();
// 初始化分配策略
BooleanPool pool = GenerateHashTreeUtil.isResourcePool(request.getConfig().getResourcePoolId());
if (pool.isPool()) {
resources = GenerateHashTreeUtil.setPoolResource(request.getConfig().getResourcePoolId());
SmoothWeighted.setServerConfig(request.getConfig().getResourcePoolId(), redisTemplate);
}
for (String reportId : executeQueue.keySet()) {
RunModeDataDTO dataDTO = executeQueue.get(reportId);
@ -46,10 +47,10 @@ public class ApiScenarioParallelService {
LoggerUtil.debug("Scenario run-开始并发执行：" + JSON.toJSONString(request));
}
// 本地执行生成hashTree
if (request.getConfig() != null && !runRequest.getPool().isPool()) {
if (!pool.isPool()) {
runRequest.setHashTree(GenerateHashTreeUtil.generateHashTree(dataDTO.getScenario(), dataDTO.getPlanEnvMap(), runRequest));
}
jMeterService.run(runRequest, resources);
jMeterService.run(runRequest);
}
}
}

20
backend/src/main/java/io/metersphere/api/exec/scenario/ApiScenarioSerialService.java
View File

@ -15,6 +15,7 @@ import io.metersphere.api.dto.definition.request.sampler.MsJDBCSampler;
import io.metersphere.api.dto.definition.request.sampler.MsTCPSampler;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.api.jmeter.JMeterService;
import io.metersphere.api.jmeter.utils.SmoothWeighted;
import io.metersphere.api.service.ApiExecutionQueueService;
import io.metersphere.api.service.ApiTestEnvironmentService;
import io.metersphere.api.service.RemakeReportService;
@ -33,10 +34,13 @@ import io.metersphere.plugin.core.MsTestElement;
import io.metersphere.utils.LoggerUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.jorphan.collections.HashTree;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.*;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.Map;
@Service
public class ApiScenarioSerialService {
@ -58,6 +62,8 @@ public class ApiScenarioSerialService {
private ApiScenarioEnvService apiScenarioEnvService;
@Resource
private ObjectMapper mapper;
@Resource
private RedisTemplate<String, Object> redisTemplate;
public void serial(ApiExecutionQueue executionQueue, ApiExecutionQueueDetail queue) {
LoggerUtil.debug("Scenario run-执行脚本装载-进入串行准备");
@ -126,13 +132,11 @@ public class ApiScenarioSerialService {
if (queue != null) {
runRequest.setPlatformUrl(queue.getId());
}
List<TestResource> resources = new ArrayList<>();
if (runRequest.getPool().isPool()) {
resources = GenerateHashTreeUtil.setPoolResource(runRequest.getPoolId());
SmoothWeighted.setServerConfig(runRequest.getPoolId(), redisTemplate);
}
// 开始执行
jMeterService.run(runRequest, resources);
jMeterService.run(runRequest);
} catch (Exception e) {
RemakeReportService remakeReportService = CommonBeanFactory.getBean(RemakeReportService.class);
remakeReportService.remake(runRequest);
@ -206,7 +210,7 @@ public class ApiScenarioSerialService {
list.addAll(elements);
}
if (element.getString("type").equals("HTTPSamplerProxy")) {
MsHTTPSamplerProxy httpSamplerProxy = JSON.parseObject(api, MsHTTPSamplerProxy.class,Feature.DisableSpecialKeyDetect);
MsHTTPSamplerProxy httpSamplerProxy = JSON.parseObject(api, MsHTTPSamplerProxy.class, Feature.DisableSpecialKeyDetect);
httpSamplerProxy.setHashTree(list);
httpSamplerProxy.setName(planId);
if (StringUtils.isNotEmpty(envId)) {
@ -215,7 +219,7 @@ public class ApiScenarioSerialService {
return httpSamplerProxy;
}
if (element.getString("type").equals("TCPSampler")) {
MsTCPSampler msTCPSampler = JSON.parseObject(api, MsTCPSampler.class,Feature.DisableSpecialKeyDetect);
MsTCPSampler msTCPSampler = JSON.parseObject(api, MsTCPSampler.class, Feature.DisableSpecialKeyDetect);
if (StringUtils.isNotEmpty(envId)) {
msTCPSampler.setUseEnvironment(envId);
}
@ -224,7 +228,7 @@ public class ApiScenarioSerialService {
return msTCPSampler;
}
if (element.getString("type").equals("DubboSampler")) {
MsDubboSampler dubboSampler = JSON.parseObject(api, MsDubboSampler.class,Feature.DisableSpecialKeyDetect);
MsDubboSampler dubboSampler = JSON.parseObject(api, MsDubboSampler.class, Feature.DisableSpecialKeyDetect);
if (StringUtils.isNotEmpty(envId)) {
dubboSampler.setUseEnvironment(envId);
}

4
backend/src/main/java/io/metersphere/api/exec/utils/GenerateHashTreeUtil.java
View File

@ -14,7 +14,6 @@ import io.metersphere.api.jmeter.ResourcePoolCalculation;
import io.metersphere.api.service.ApiExecutionQueueService;
import io.metersphere.api.service.RemakeReportService;
import io.metersphere.base.domain.ApiScenarioWithBLOBs;
import io.metersphere.base.domain.TestResource;
import io.metersphere.base.domain.TestResourcePool;
import io.metersphere.base.mapper.TestResourcePoolMapper;
import io.metersphere.commons.constants.ResourcePoolTypeEnum;
@ -23,6 +22,7 @@ import io.metersphere.commons.utils.CommonBeanFactory;
import io.metersphere.commons.utils.LogUtil;
import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.dto.JvmInfoDTO;
import io.metersphere.dto.ResultDTO;
import io.metersphere.dto.RunModeConfigDTO;
import io.metersphere.plugin.core.MsTestElement;
@ -98,7 +98,7 @@ public class GenerateHashTreeUtil {
return pool;
}
public static List<TestResource> setPoolResource(String id) {
public static List<JvmInfoDTO> setPoolResource(String id) {
if (GenerateHashTreeUtil.isResourcePool(id).isPool() && !GenerateHashTreeUtil.isResourcePool(id).isK8s()) {
ResourcePoolCalculation resourcePoolCalculation = CommonBeanFactory.getBean(ResourcePoolCalculation.class);
return resourcePoolCalculation.getPools(id);

59
backend/src/main/java/io/metersphere/api/jmeter/JMeterService.java
View File

@ -3,6 +3,8 @@ package io.metersphere.api.jmeter;
import com.alibaba.fastjson.JSON;
import io.metersphere.api.exec.queue.ExecThreadPoolExecutor;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.api.jmeter.utils.ServerConfig;
import io.metersphere.api.jmeter.utils.SmoothWeighted;
import io.metersphere.api.service.ApiScenarioReportService;
import io.metersphere.api.service.RemakeReportService;
import io.metersphere.base.domain.TestResource;
@ -14,6 +16,7 @@ import io.metersphere.config.KafkaConfig;
import io.metersphere.constants.RunModeConstants;
import io.metersphere.dto.BaseSystemConfigDTO;
import io.metersphere.dto.JmeterRunRequestDTO;
import io.metersphere.dto.JvmInfoDTO;
import io.metersphere.dto.NodeDTO;
import io.metersphere.jmeter.JMeterBase;
import io.metersphere.jmeter.LocalRunner;
@ -29,6 +32,7 @@ import org.apache.jmeter.testelement.TestElement;
import org.apache.jmeter.util.JMeterUtils;
import org.apache.jorphan.collections.HashTree;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
@ -45,6 +49,8 @@ public class JMeterService {
private JmeterProperties jmeterProperties;
@Resource
private RestTemplate restTemplate;
@Resource
private RedisTemplate<String, Object> redisTemplate;
@PostConstruct
private void init() {
@ -114,7 +120,7 @@ public class JMeterService {
runner.run(request.getReportId());
}
private void runNode(JmeterRunRequestDTO request, List<TestResource> resources) {
private void runNode(JmeterRunRequestDTO request) {
// 获取可以执行的资源池
BaseSystemConfigDTO baseInfo = CommonBeanFactory.getBean(SystemParameterService.class).getBaseInfo();
// 占位符
@ -147,38 +153,34 @@ public class JMeterService {
MSException.throwException(e.getMessage());
}
} else {
this.send(request, resources);
this.send(request);
}
}
private void send(JmeterRunRequestDTO request, List<TestResource> resources) {
private void send(JmeterRunRequestDTO request) {
try {
if (StringUtils.isNotEmpty(request.getPoolId()) && CollectionUtils.isEmpty(resources)) {
resources = GenerateHashTreeUtil.setPoolResource(request.getPoolId());
if (redisTemplate.opsForValue().get(SmoothWeighted.EXEC_INDEX + request.getPoolId()) != null) {
long index = Long.parseLong(redisTemplate.opsForValue().get(SmoothWeighted.EXEC_INDEX + request.getPoolId()).toString());
redisTemplate.opsForValue().set(SmoothWeighted.EXEC_INDEX + request.getPoolId(), index + 1);
}
if (CollectionUtils.isEmpty(resources)) {
ServerConfig config = SmoothWeighted.calculate(request.getPoolId(), redisTemplate);
if (config == null) {
config = SmoothWeighted.getResource(request.getPoolId());
}
if (config == null) {
LoggerUtil.info("未获取到资源池，请检查配置【系统设置-系统-测试资源池】");
RemakeReportService remakeReportService = CommonBeanFactory.getBean(RemakeReportService.class);
remakeReportService.remake(request);
return;
}
int index = (int) (Math.random() * resources.size());
TestResource testResource = resources.get(index);
String configuration = testResource.getConfiguration();
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
request.setCorePoolSize(node.getMaxConcurrency());
request.setEnable(node.isEnable());
String nodeIp = node.getIp();
Integer port = node.getPort();
String uri = String.format(BASE_URL + "/jmeter/api/start", nodeIp, port);
LoggerUtil.info("开始发送请求【 " + request.getReportId() + " 】,资源【 " + request.getTestId() + " 】" + uri + " 节点执行");
ResponseEntity<String> result = restTemplate.postForEntity(uri, request, String.class);
request.setCorePoolSize(config.getCorePoolSize());
request.setEnable(config.isEnable());
LoggerUtil.info("开始发送请求【 " + request.getReportId() + " 】,资源【 " + request.getTestId() + " 】" + config.getUrl() + " 节点执行");
ResponseEntity<String> result = restTemplate.postForEntity(config.getUrl(), request, String.class);
if (result == null || !StringUtils.equals("SUCCESS", result.getBody())) {
RemakeReportService remakeReportService = CommonBeanFactory.getBean(RemakeReportService.class);
remakeReportService.remake(request);
LoggerUtil.error("发送请求[ " + request.getTestId() + " ] 到" + uri + " 节点执行失败");
LoggerUtil.error("发送请求[ " + request.getTestId() + " ] 到" + config.getUrl() + " 节点执行失败");
LoggerUtil.info(result.getBody());
}
} catch (Exception e) {
@ -190,9 +192,18 @@ public class JMeterService {
}
public void run(JmeterRunRequestDTO request) {
if (request.getPool().isPool()) {
this.runNode(request);
} else {
CommonBeanFactory.getBean(ExecThreadPoolExecutor.class).addTask(request);
}
}
@Deprecated
public void run(JmeterRunRequestDTO request, List<TestResource> resources) {
if (request.getPool().isPool()) {
this.runNode(request, resources);
this.runNode(request);
} else {
CommonBeanFactory.getBean(ExecThreadPoolExecutor.class).addTask(request);
}
@ -204,13 +215,13 @@ public class JMeterService {
public boolean getRunningQueue(String poolId, String reportId) {
try {
List<TestResource> resources = GenerateHashTreeUtil.setPoolResource(poolId);
List<JvmInfoDTO> resources = GenerateHashTreeUtil.setPoolResource(poolId);
if (CollectionUtils.isEmpty(resources)) {
return false;
}
boolean isRunning = false;
for (TestResource testResource : resources) {
String configuration = testResource.getConfiguration();
for (JvmInfoDTO testResource : resources) {
String configuration = testResource.getTestResource().getConfiguration();
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
String nodeIp = node.getIp();
Integer port = node.getPort();

23
backend/src/main/java/io/metersphere/api/jmeter/ResourcePoolCalculation.java
View File

@ -7,9 +7,11 @@ import io.metersphere.base.domain.TestResourcePool;
import io.metersphere.base.domain.TestResourcePoolExample;
import io.metersphere.base.mapper.TestResourceMapper;
import io.metersphere.base.mapper.TestResourcePoolMapper;
import io.metersphere.commons.utils.BeanUtils;
import io.metersphere.dto.JvmInfoDTO;
import io.metersphere.dto.NodeDTO;
import io.metersphere.dto.TestResourceDTO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.RestTemplate;
@ -31,22 +33,22 @@ public class ResourcePoolCalculation {
private static final String BASE_URL = "http://%s:%d";
private String getNodeJvmInfo(String uri) {
private JvmInfoDTO getNodeJvmInfo(String uri) {
try {
return restTemplate.getForObject(uri, String.class);
return restTemplate.getForObject(uri, JvmInfoDTO.class);
} catch (Exception e) {
return null;
}
}
public List<TestResource> getPools(String resourcePoolId) {
public List<JvmInfoDTO> getPools(String resourcePoolId) {
// 获取可以执行的资源池
TestResourcePoolExample example = new TestResourcePoolExample();
example.createCriteria().andStatusEqualTo("VALID").andTypeEqualTo("NODE").andIdEqualTo(resourcePoolId);
List<TestResourcePool> pools = testResourcePoolMapper.selectByExample(example);
// 按照NODE节点的可用内存空间大小排序
List<TestResource> availableNodes = new ArrayList<>();
List<JvmInfoDTO> availableNodes = new ArrayList<>();
if (CollectionUtils.isNotEmpty(pools)) {
List<String> poolIds = pools.stream().map(pool -> pool.getId()).collect(Collectors.toList());
TestResourceExample resourceExample = new TestResourceExample();
@ -57,12 +59,15 @@ public class ResourcePoolCalculation {
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
String nodeIp = node.getIp();
Integer port = node.getPort();
String uri = String.format(BASE_URL + "/jmeter/status", nodeIp, port);
String status = this.getNodeJvmInfo(uri);
if (!StringUtils.equals(status, "OK")) {
String uri = String.format(BASE_URL + "/jmeter/getJvmInfo", nodeIp, port);
JvmInfoDTO nodeJvm = this.getNodeJvmInfo(uri);
if (nodeJvm == null) {
continue;
}
availableNodes.add(testResource);
TestResourceDTO dto = new TestResourceDTO();
BeanUtils.copyBean(dto, testResource);
nodeJvm.setTestResource(dto);
availableNodes.add(nodeJvm);
}
}
return availableNodes;

68
backend/src/main/java/io/metersphere/api/jmeter/utils/ServerConfig.java Normal file
View File

@ -0,0 +1,68 @@
package io.metersphere.api.jmeter.utils;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import java.io.Serializable;
@Data
public class ServerConfig implements Serializable {
//服务名称
public String url;
//初始权重
public int weight;
//当前权重
public int currentWeight;
// 资源池并发数
private int corePoolSize;
// 是否开启JAR同步
private boolean enable;
public ServerConfig() {
}
public ServerConfig(String url, int weight) {
this.url = url;
this.weight = weight;
}
public ServerConfig(String url, int weight, int currentWeight) {
this.url = url;
this.weight = weight;
this.currentWeight = currentWeight;
}
public ServerConfig(String url, int weight, int currentWeight, int corePoolSize, boolean enable) {
this.url = url;
this.weight = weight;
this.currentWeight = currentWeight;
this.corePoolSize = corePoolSize;
this.enable = enable;
}
@Override
public String toString() {
return "ServerConfig{" + "url='" + url + '}';
}
@Override
public boolean equals(Object value) {
if (value != null && value instanceof ServerConfig) {
return StringUtils.equals(((ServerConfig) value).getUrl(), this.getUrl());
}
return false;
}
@Override
public ServerConfig clone() throws CloneNotSupportedException {
ServerConfig serverConfig = new ServerConfig();
serverConfig.setCurrentWeight(this.currentWeight);
serverConfig.setUrl(this.url);
serverConfig.setWeight(this.weight);
return serverConfig;
}
}

130
backend/src/main/java/io/metersphere/api/jmeter/utils/SmoothWeighted.java Normal file
View File

@ -0,0 +1,130 @@
package io.metersphere.api.jmeter.utils;
import com.alibaba.fastjson.JSON;
import io.metersphere.api.exec.utils.GenerateHashTreeUtil;
import io.metersphere.dto.JvmInfoDTO;
import io.metersphere.dto.NodeDTO;
import io.metersphere.dto.TestResourceDTO;
import io.metersphere.utils.LoggerUtil;
import io.metersphere.vo.BooleanPool;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.redis.core.RedisTemplate;
import java.util.*;
/**
* 平滑加权轮询算法
*/
public class SmoothWeighted {
private static final String BASE_URL = "http://%s:%d";
public static final String CONFIG = "CONFIG_";
public static final String EXEC_INDEX = "EXEC_INDEX_";
public static void setServerConfig(String poolId, RedisTemplate client) {
if (StringUtils.isNotEmpty(poolId)) {
return;
}
List<JvmInfoDTO> resources = new ArrayList<>();
BooleanPool pool = GenerateHashTreeUtil.isResourcePool(poolId);
if (pool.isPool()) {
resources = GenerateHashTreeUtil.setPoolResource(poolId);
}
if (CollectionUtils.isEmpty(resources)) {
client.delete(CONFIG + poolId);
client.delete(EXEC_INDEX + poolId);
return;
}
Map<String, ServerConfig> configs = new HashMap<>();
for (JvmInfoDTO jvmInfoDTO : resources) {
String configuration = jvmInfoDTO.getTestResource().getConfiguration();
if (StringUtils.isNotEmpty(configuration)) {
NodeDTO node = JSON.parseObject(configuration, NodeDTO.class);
String uri = String.format(BASE_URL + "/jmeter/api/start", node.getIp(), node.getPort());
configs.put(uri, new ServerConfig(uri, 1, 1, node.getMaxConcurrency(), node.isEnable()));
}
}
if (client.opsForValue().get(CONFIG + poolId) == null) {
client.opsForValue().set(CONFIG + poolId, new ArrayList<>(configs.values()));
} else {
// 合并相同节点信息，更改权重
List<ServerConfig> serverConfigs = (List<ServerConfig>) client.opsForValue().get(CONFIG + poolId);
serverConfigs.forEach(item -> {
if (configs.containsKey(item.getUrl())) {
item.setCorePoolSize(configs.get(item.getUrl()).getCorePoolSize());
item.setEnable(configs.get(item.getUrl()).isEnable());
item.setWeight(configs.get(item.getUrl()).getWeight());
}
});
// 添加新增节点
configs.forEach((k, v) -> {
if (!serverConfigs.contains(v)) {
serverConfigs.add(v);
}
});
// 异常已经废弃的节点
serverConfigs.removeIf(item -> !configs.containsKey(item.getUrl()));
client.opsForValue().set(CONFIG + poolId, serverConfigs);
}
if (client.opsForValue().get(EXEC_INDEX + poolId) == null) {
client.opsForValue().set(EXEC_INDEX + poolId, 1);
}
}
public static ServerConfig calculate(String poolId, RedisTemplate client) {
if (client.opsForValue().get(EXEC_INDEX + poolId) == null) {
client.opsForValue().set(EXEC_INDEX + poolId, 1);
}
List<ServerConfig> serverList = new ArrayList<>();
if (client.opsForValue().get(CONFIG + poolId) != null) {
serverList = (List<ServerConfig>) client.opsForValue().get(CONFIG + poolId);
}
// 最大权重
int weightSum = serverList.stream().map(ServerConfig::getWeight).reduce((x, y) -> x += y).get();
long execIndex = Long.parseLong(client.opsForValue().get(EXEC_INDEX + poolId).toString());
// 选出当前有效权重最大的实例，将当前有效权重currentWeight减去所有实例的"权重和"（weightSum），且变量tmpSv指向此位置；
ServerConfig max = Collections.max(serverList, Comparator.comparingLong(ServerConfig::getCurrentWeight));
// 选中的实例
ServerConfig tmpConfig = null;
for (ServerConfig serverConfig : serverList) {
if (max.equals(serverConfig)) {
serverConfig.setCurrentWeight(serverConfig.getCurrentWeight() - weightSum);
if (tmpConfig == null || serverConfig.getCurrentWeight() > tmpConfig.getCurrentWeight()) {
tmpConfig = serverConfig;
}
}
//将每个实例的当前有效权重currentWeight都加上配置权重weight；
serverConfig.setCurrentWeight(serverConfig.getCurrentWeight() + serverConfig.getWeight());
}
// 选中前的当前权重
LoggerUtil.info("第" + (execIndex) + "次选中前的当前权重:" + serverList.toString());
if (client.opsForValue().get(CONFIG + poolId) != null) {
client.opsForValue().set(CONFIG + poolId, serverList);
}
return tmpConfig;
}
public static ServerConfig getResource(String poolId) {
BooleanPool pool = GenerateHashTreeUtil.isResourcePool(poolId);
if (pool.isPool()) {
List<JvmInfoDTO> resources = GenerateHashTreeUtil.setPoolResource(poolId);
if (CollectionUtils.isNotEmpty(resources)) {
int index = (int) (Math.random() * resources.size());
TestResourceDTO testResource = resources.get(index).getTestResource();
NodeDTO node = JSON.parseObject(testResource.getConfiguration(), NodeDTO.class);
String nodeIp = node.getIp();
Integer port = node.getPort();
String uri = String.format(BASE_URL + "/jmeter/api/start", nodeIp, port);
return new ServerConfig(uri, 1, 1, node.getMaxConcurrency(), node.isEnable());
}
}
return null;
}
}

2
backend/src/main/java/io/metersphere/api/service/ApiAutomationService.java
View File

@ -10,7 +10,6 @@ import io.metersphere.api.dto.automation.parse.ApiScenarioImportUtil;
import io.metersphere.api.dto.automation.parse.ScenarioImport;
import io.metersphere.api.dto.automation.parse.ScenarioImportParserFactory;
import io.metersphere.api.dto.datacount.ApiDataCountResult;
import io.metersphere.api.dto.datacount.ApiMethodUrlDTO;
import io.metersphere.api.dto.definition.ApiTestCaseInfo;
import io.metersphere.api.dto.definition.RunDefinitionRequest;
import io.metersphere.api.dto.definition.request.*;
@ -459,7 +458,6 @@ public class ApiAutomationService {
scenario.setUpdateTime(System.currentTimeMillis());
scenario.setDescription(request.getDescription());
scenario.setCreateUser(SessionUtils.getUserId());
scenario.setScenarioDefinition(JSON.toJSONString(request.getScenarioDefinition()));
Boolean isValidEnum = EnumUtils.isValidEnum(EnvironmentType.class, request.getEnvironmentType());
if (BooleanUtils.isTrue(isValidEnum)) {