test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix(系统设置): 优化系统参数设置保存时更改mock环境的方法 

--bug=1024524 --user=宋天阳 【系统设置】项目及资源池较多时，修改系统参数设置会发生OOM
https://www.tapd.cn/55049933/s/1352042
This commit is contained in:
song-tianyang 2023-03-17 15:16:55 +08:00 committed by 建国
parent fec33d390e
commit 7dac2c6684
7 changed files with 194 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
api-test/backend/src/main/java/io/metersphere/listener/CheckMockEnvironmentListener.java
View File

@ -1,29 +0,0 @@
package io.metersphere.listener;
import io.metersphere.commons.constants.KafkaTopicConstants;
import io.metersphere.commons.utils.LogUtil;
import io.metersphere.environment.service.BaseEnvironmentService;
import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;
import jakarta.annotation.Resource;
@Component
public class CheckMockEnvironmentListener {
public static final String CONSUME_ID = "check-mock-env";
@Resource
private BaseEnvironmentService baseEnvironmentService;
@KafkaListener(id = CONSUME_ID, topics = KafkaTopicConstants.CHECK_MOCK_ENV_TOPIC, groupId = "${spring.application.name}")
public void consume(ConsumerRecord<?, String> record) {
String baseUrl = record.value();
LogUtil.info("consume check-mock-env message, base url: " + baseUrl);
if (StringUtils.isNotBlank(baseUrl)) {
baseEnvironmentService.checkMockEvnInfoByBaseUrl(baseUrl);
}
}
}

13
framework/sdk-parent/sdk/src/main/java/io/metersphere/base/mapper/ext/ExtApiTestEnvironmentMapper.java Normal file
View File

@ -0,0 +1,13 @@
package io.metersphere.base.mapper.ext;
import io.metersphere.base.domain.ApiTestEnvironmentWithBLOBs;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import java.util.List;
public interface ExtApiTestEnvironmentMapper {
@Select("SELECT * FROM api_test_environment WHERE name = #{envName} LIMIT #{startNum},#{pageSize}")
List<ApiTestEnvironmentWithBLOBs> selectByNameAndLimitNum(@Param("envName") String envName, @Param("startNum") int startNum, @Param("pageSize") int pageSize);
}

2
framework/sdk-parent/sdk/src/main/java/io/metersphere/commons/constants/KafkaTopicConstants.java
View File

@ -10,8 +10,6 @@ public interface KafkaTopicConstants {
String DEBUG_TOPICS = "MS-API-DEBUG-TOPIC";
// 测试计划接收执行结果TOPIC
String TEST_PLAN_REPORT_TOPIC = "TEST_PLAN_REPORT_TOPIC";
// 保存当前站点时检查MOCK环境
String CHECK_MOCK_ENV_TOPIC = "CHECK_MOCK_ENV_TOPIC";
// 上传插件后，通知各服务从 minio 加载插件, 删除插件后卸载插件
String PLATFORM_PLUGIN = "PLATFORM_PLUGIN";
}

12
framework/sdk-parent/sdk/src/main/java/io/metersphere/environment/dto/ApiMockEnvUpdateDTO.java Normal file
View File

@ -0,0 +1,12 @@
package io.metersphere.environment.dto;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class ApiMockEnvUpdateDTO {
private String baseUrl;
private int limitStart;
private int limitSize;
}

127
framework/sdk-parent/sdk/src/main/java/io/metersphere/environment/service/BaseEnvironmentService.java
View File

@ -4,6 +4,7 @@ import io.metersphere.base.domain.*;
import io.metersphere.base.mapper.*;
import io.metersphere.base.mapper.ext.BaseApiTestEnvironmentMapper;
import io.metersphere.base.mapper.ext.BaseEnvironmentGroupMapper;
import io.metersphere.base.mapper.ext.ExtApiTestEnvironmentMapper;
import io.metersphere.commons.constants.FileAssociationType;
import io.metersphere.commons.constants.ProjectApplicationType;
import io.metersphere.commons.exception.MSException;
@ -11,6 +12,7 @@ import io.metersphere.commons.utils.*;
import io.metersphere.dto.BaseSystemConfigDTO;
import io.metersphere.dto.ProjectConfig;
import io.metersphere.environment.dto.*;
import io.metersphere.environment.utils.BatchProcessingUtil;
import io.metersphere.i18n.Translator;
import io.metersphere.log.utils.ReflexObjectUtil;
import io.metersphere.log.vo.DetailColumn;
@ -23,6 +25,7 @@ import io.metersphere.service.BaseProjectApplicationService;
import io.metersphere.service.BaseProjectService;
import io.metersphere.service.NodeTreeService;
import io.metersphere.service.SystemParameterService;
import jakarta.annotation.Resource;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.BooleanUtils;
@ -38,7 +41,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import jakarta.annotation.Resource;
import java.util.*;
import java.util.stream.Collectors;
@ -54,6 +56,8 @@ public class BaseEnvironmentService extends NodeTreeService<ApiModuleDTO> {
@Resource
private BaseEnvironmentGroupMapper baseEnvironmentGroupMapper;
@Resource
private ExtApiTestEnvironmentMapper extApiTestEnvironmentMapper;
@Resource
private SqlSessionFactory sqlSessionFactory;
@Resource
private EnvironmentGroupProjectMapper environmentGroupProjectMapper;
@ -793,19 +797,120 @@ public class BaseEnvironmentService extends NodeTreeService<ApiModuleDTO> {
}
public void checkMockEvnInfoByBaseUrl(String baseUrl) {
List<ApiTestEnvironmentWithBLOBs> allEvnList = this.selectByExampleWithBLOBs(null);
for (ApiTestEnvironmentWithBLOBs model : allEvnList) {
if (StringUtils.equals(model.getName(), "Mock环境")) {
String protocal = "";
if (baseUrl.startsWith("http:")) {
protocal = "http";
} else if (baseUrl.startsWith("https:")) {
protocal = "https";
ApiTestEnvironmentExample example = new ApiTestEnvironmentExample();
example.createCriteria().andNameEqualTo(MOCK_EVN_NAME);
long mockEnvCount = apiTestEnvironmentMapper.countByExample(example);
BatchProcessingUtil.batchUpdateMockEnvConfig(baseUrl, mockEnvCount, this::updateMockEvnInfoByBaseUrlAndLimitNum);
}
private void updateMockEvnInfoByBaseUrlAndLimitNum(ApiMockEnvUpdateDTO mockEnvUpdateDTO) {
if (StringUtils.isNotEmpty(mockEnvUpdateDTO.getBaseUrl())) {
List<ApiTestEnvironmentWithBLOBs> allEvnList = extApiTestEnvironmentMapper.selectByNameAndLimitNum(
MOCK_EVN_NAME, mockEnvUpdateDTO.getLimitStart(), mockEnvUpdateDTO.getLimitSize());
List<ApiTestEnvironmentWithBLOBs> updateList = new ArrayList<>();
for (ApiTestEnvironmentWithBLOBs model : allEvnList) {
if (StringUtils.equals(model.getName(), MOCK_EVN_NAME)) {
String protocal = "";
if (mockEnvUpdateDTO.getBaseUrl().startsWith("http:")) {
protocal = "http";
} else if (mockEnvUpdateDTO.getBaseUrl().startsWith("https:")) {
protocal = "https";
}
ApiTestEnvironmentWithBLOBs updateModel = this.updateHttpMockEvn(model, protocal, mockEnvUpdateDTO.getBaseUrl());
updateList.add(updateModel);
}
String projectNumber = this.getSystemIdByProjectId(model.getProjectId());
this.checkMockEvnIsRightful(model, protocal, model.getProjectId(), projectNumber, model.getName(), baseUrl);
}
if (CollectionUtils.isNotEmpty(updateList)) {
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
ApiTestEnvironmentMapper batchMapper = sqlSession.getMapper(ApiTestEnvironmentMapper.class);
for (ApiTestEnvironmentWithBLOBs updateModel : updateList) {
batchMapper.updateByPrimaryKeyWithBLOBs(updateModel);
}
sqlSession.flushStatements();
if (sqlSession != null && sqlSessionFactory != null) {
SqlSessionUtils.closeSqlSession(sqlSession, sqlSessionFactory);
}
}
}
}
private ApiTestEnvironmentWithBLOBs updateHttpMockEvn(ApiTestEnvironmentWithBLOBs mockEnv, String protocol, String newUrl) {
ApiTestEnvironmentWithBLOBs updatedEnv = null;
if (mockEnv.getConfig() != null) {
try {
com.alibaba.fastjson.JSONObject configObj = com.alibaba.fastjson.JSONObject.parseObject(mockEnv.getConfig());
if (configObj.containsKey("httpConfig")) {
com.alibaba.fastjson.JSONObject httpObj = configObj.getJSONObject("httpConfig");
if (httpObj.containsKey("isMock") && httpObj.getBoolean("isMock")) {
if (httpObj.containsKey("conditions")) {
String url = newUrl;
if (url.startsWith("http://")) {
url = url.substring(7);
} else if (url.startsWith("https://")) {
url = url.substring(8);
}
String ipStr = url;
String portStr = "";
if (url.contains(":") && !url.endsWith(":")) {
String[] urlArr = url.split(":");
int port = -1;
try {
port = Integer.parseInt(urlArr[urlArr.length - 1]);
} catch (Exception e) {
}
if (port > -1) {
portStr = String.valueOf(port);
ipStr = urlArr[0];
}
}
if (httpObj.containsKey("socket") && httpObj.containsKey("protocol") && httpObj.containsKey("port")) {
String httpSocket = httpObj.getString("socket");
if (httpSocket.contains("/mock")) {
String[] httpSocketArr = StringUtils.split(httpSocket, "/mock");
httpSocketArr[0] = url;
httpSocket = StringUtils.join(httpSocketArr);
}
httpObj.put("socket", httpSocket);
httpObj.put("protocol", protocol);
if (StringUtils.isNotEmpty(portStr)) {
httpObj.put("port", portStr);
} else {
httpObj.put("port", "");
}
}
com.alibaba.fastjson.JSONArray conditions = httpObj.getJSONArray("conditions");
for (int i = 0; i < conditions.size(); i++) {
com.alibaba.fastjson.JSONObject httpItem = conditions.getJSONObject(i);
if (httpItem.containsKey("socket") && httpItem.containsKey("protocol") && httpItem.containsKey("domain")) {
String httpSocket = httpItem.getString("socket");
if (httpSocket.contains("/mock")) {
String[] httpSocketArr = StringUtils.split(httpSocket, "/mock");
httpSocketArr[0] = url;
httpSocket = StringUtils.join(httpSocketArr);
}
httpItem.put("socket", httpSocket);
httpItem.put("domain", ipStr);
httpItem.put("protocol", protocol);
if (StringUtils.isNotEmpty(portStr)) {
httpItem.put("port", portStr);
} else {
httpItem.put("port", "");
}
}
}
updatedEnv = mockEnv;
updatedEnv.setConfig(configObj.toJSONString());
}
}
}
} catch (Exception e) {
LogUtil.error(e);
}
}
return updatedEnv;
}
public LinkedHashMap<String, List<String>> selectProjectNameAndEnvName(Map<String, List<String>> projectEnvIdMap) {

24
framework/sdk-parent/sdk/src/main/java/io/metersphere/environment/utils/BatchProcessingUtil.java Normal file
View File

@ -0,0 +1,24 @@
package io.metersphere.environment.utils;
import io.metersphere.environment.dto.ApiMockEnvUpdateDTO;
import java.util.function.Consumer;
public class BatchProcessingUtil {
public static final int BATCH_PROCESS_QUANTITY = 1000;
public static void batchUpdateMockEnvConfig(String newUrl, long mockEnvCount, Consumer<ApiMockEnvUpdateDTO> func) {
long remainderMockEnvCount = mockEnvCount;
int startPage = 0;
while (remainderMockEnvCount > 0) {
ApiMockEnvUpdateDTO dto = new ApiMockEnvUpdateDTO();
dto.setBaseUrl(newUrl);
dto.setLimitSize(BATCH_PROCESS_QUANTITY);
dto.setLimitStart(startPage);
func.accept(dto);
startPage = startPage + BATCH_PROCESS_QUANTITY;
remainderMockEnvCount = remainderMockEnvCount - BATCH_PROCESS_QUANTITY;
}
}
}

45
framework/sdk-parent/sdk/src/main/java/io/metersphere/service/SystemParameterService.java
View File

@ -5,7 +5,6 @@ import io.metersphere.base.mapper.SystemHeaderMapper;
import io.metersphere.base.mapper.SystemParameterMapper;
import io.metersphere.base.mapper.UserHeaderMapper;
import io.metersphere.base.mapper.ext.BaseSystemParameterMapper;
import io.metersphere.commons.constants.KafkaTopicConstants;
import io.metersphere.commons.constants.MicroServiceName;
import io.metersphere.commons.constants.ParamConstants;
import io.metersphere.commons.exception.MSException;
@ -13,6 +12,7 @@ import io.metersphere.commons.utils.EncryptUtils;
import io.metersphere.commons.utils.JSON;
import io.metersphere.commons.utils.LogUtil;
import io.metersphere.dto.BaseSystemConfigDTO;
import io.metersphere.environment.service.BaseEnvironmentService;
import io.metersphere.i18n.Translator;
import io.metersphere.ldap.domain.LdapInfo;
import io.metersphere.log.utils.ReflexObjectUtil;
@ -24,15 +24,14 @@ import io.metersphere.notice.domain.Receiver;
import io.metersphere.notice.sender.NoticeModel;
import io.metersphere.notice.sender.impl.MailNoticeSender;
import io.metersphere.request.HeaderRequest;
import jakarta.annotation.Resource;
import jakarta.mail.MessagingException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.mail.javamail.JavaMailSenderImpl;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import jakarta.annotation.Resource;
import jakarta.mail.MessagingException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@ -52,11 +51,10 @@ public class SystemParameterService {
private SystemHeaderMapper systemHeaderMapper;
@Resource
private MailNoticeSender mailNoticeSender;
@Resource
private BaseEnvironmentService baseEnvironmentService;
@Resource
private MicroService microService;
@Resource
private KafkaTemplate<String, String> kafkaTemplate;
public String searchEmail() {
return baseSystemParameterMapper.email();
@ -266,16 +264,31 @@ public class SystemParameterService {
}
// 去掉路径最后的 /
param.setParamValue(StringUtils.removeEnd(param.getParamValue(), "/"));
example.createCriteria().andParamKeyEqualTo(param.getParamKey());
if (systemParameterMapper.countByExample(example) > 0) {
systemParameterMapper.updateByPrimaryKey(param);
} else {
systemParameterMapper.insert(param);
}
example.clear();
if (StringUtils.equals(param.getParamKey(), "base.url")) {
kafkaTemplate.send(KafkaTopicConstants.CHECK_MOCK_ENV_TOPIC, param.getParamValue());
example.createCriteria().andParamKeyEqualTo(param.getParamKey());
List<SystemParameter> baseUrlParameterList = systemParameterMapper.selectByExample(example);
String oldBaseUrl = null;
if (CollectionUtils.isNotEmpty(baseUrlParameterList)) {
SystemParameter parameter = baseUrlParameterList.get(0);
if (!StringUtils.equals(parameter.getParamValue(), param.getParamValue())) {
oldBaseUrl = parameter.getParamValue();
systemParameterMapper.updateByPrimaryKey(param);
}
} else {
systemParameterMapper.insert(param);
}
example.clear();
if (StringUtils.isNotEmpty(oldBaseUrl)) {
baseEnvironmentService.checkMockEvnInfoByBaseUrl(param.getParamValue());
}
} else {
example.createCriteria().andParamKeyEqualTo(param.getParamKey());
if (systemParameterMapper.countByExample(example) > 0) {
systemParameterMapper.updateByPrimaryKey(param);
} else {
systemParameterMapper.insert(param);
}
example.clear();
}
});
}