test-version22
/
MeterSphere
mirror of https://gitee.com/fit2cloud-feizhiyun/MeterSphere.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

build: 批量插入

This commit is contained in:
CaptainB 2023-05-24 12:17:17 +08:00 committed by 刘瑞斌
parent 5fe115c70b
commit bf2b874fb5
7 changed files with 257 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.java
View File

@ -2,6 +2,7 @@ package io.metersphere.sdk.mapper;
import io.metersphere.sdk.dto.UserDTO;
import io.metersphere.system.domain.User;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@ -9,4 +10,8 @@ public interface UserMapper {
UserDTO selectById(String id);
List<User> findAll();
void insert(User user);
void batchSave(@Param("users") List<User> users);
}

17
backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.xml
View File

@ -13,4 +13,21 @@
SELECT *
FROM user
</select>
<insert id="insert">
INSERT INTO user(id, name, email, password, status, create_time, update_time, language, last_workspace_id, phone,
source, last_project_id, create_user)
VALUES (#{id}, #{name}, #{email}, #{password}, #{status}, #{createTime}, #{updateTime}, #{language},
#{lastWorkspaceId}, #{phone}, #{source}, #{lastProjectId}, #{createUser})
</insert>
<insert id="batchSave">
INSERT INTO user(id, name, email, password, status, create_time, update_time, language, last_workspace_id, phone,
source, last_project_id, create_user)
VALUES
<foreach collection="users" item="user" separator=",">
(#{user.id}, #{user.name}, #{user.email}, #{user.password}, #{user.status}, #{user.createTime}, #{user.updateTime}, #{user.language},
#{user.lastWorkspaceId}, #{user.phone}, #{user.source}, #{user.lastProjectId}, #{user.createUser})
</foreach>
</insert>
</mapper>

21
backend/services/system-setting/src/main/java/io/metersphere/system/controller/UserController.java
View File

@ -31,4 +31,25 @@ public class UserController {
public boolean addUser(@Validated({Created.class}) @RequestBody UserDTO user) {
return userService.save(user);
}
@PostMapping("/batch-add")
public boolean batchSaveUser(@Validated({Created.class}) @RequestBody List<User> user) {
return userService.batchSave(user);
}
@PostMapping("/batch-add2")
public boolean batchSaveUser2(@Validated({Created.class}) @RequestBody List<User> user) {
return userService.batchSave2(user);
}
@PostMapping("/batch-add3")
public boolean batchSaveUser3(@Validated({Created.class}) @RequestBody List<User> user) {
return userService.batchSave3(user);
}
@GetMapping("/count")
public long batchSaveUser() {
return userService.count();
}
}

62
backend/services/system-setting/src/main/java/io/metersphere/system/service/UserService.java
View File

@ -5,9 +5,11 @@ import io.metersphere.sdk.mapper.UserMapper;
import io.metersphere.sdk.util.BeanUtils;
import io.metersphere.system.domain.User;
import io.metersphere.system.domain.UserExtend;
import io.metersphere.system.util.BatchSaveUtils;
import jakarta.annotation.Resource;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.data.jdbc.core.JdbcAggregateTemplate;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@ -21,7 +23,7 @@ public class UserService {
@Resource
private JdbcAggregateTemplate jdbcAggregateTemplate;
@Resource
private JdbcTemplate jdbcTemplate;
private SqlSessionFactory sqlSessionFactory;
public boolean save(UserDTO entity) {
User user = new User();
@ -41,4 +43,60 @@ public class UserService {
public List<User> list() {
return userMapper.findAll();
}
public boolean batchSave(List<User> users) {
long start = System.currentTimeMillis();
BatchSaveUtils.batchSave(users);
System.out.println("batch save cost: " + (System.currentTimeMillis() - start) + "ms");
return true;
}
public boolean batchSave2(List<User> users) {
long start = System.currentTimeMillis();
try (SqlSession sqlSession = sqlSessionFactory.openSession()) {
UserMapper mapper = sqlSession.getMapper(UserMapper.class);
for (int i = 0, size = users.size(); i < size; i++) {
mapper.insert(users.get(i));
if (i % 100 == 0) {
sqlSession.flushStatements();
}
}
sqlSession.flushStatements();
}
System.out.println("batch save cost: " + (System.currentTimeMillis() - start) + "ms");
return true;
}
public boolean batchSave3(List<User> users) {
long start = System.currentTimeMillis();
int batchSize = 100;
int size = users.size();
int pageSize = size / batchSize;
if (pageSize == 0) {
userMapper.batchSave(users);
System.out.println("batch save cost: " + (System.currentTimeMillis() - start) + "ms");
return true;
}
for (int i = 0; i < pageSize; i++) {
int startIndex = i * batchSize;
List<User> sub = users.subList(startIndex, startIndex + batchSize);
userMapper.batchSave(sub);
}
if (size % batchSize != 0) {
int startIndex = pageSize * batchSize;
List<User> sub = users.subList(startIndex, size);
userMapper.batchSave(sub);
}
System.out.println("batch save cost: " + (System.currentTimeMillis() - start) + "ms");
return true;
}
public long count() {
return jdbcAggregateTemplate.count(User.class);
}
}

48
backend/services/system-setting/src/main/java/io/metersphere/system/util/BatchSaveUtils.java Normal file
View File

@ -0,0 +1,48 @@
package io.metersphere.system.util;
import com.google.common.base.CaseFormat;
import io.metersphere.system.domain.User;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.springframework.jdbc.core.namedparam.BeanPropertySqlParameterSource;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.stereotype.Component;
import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.List;
@Component
public class BatchSaveUtils {
private static NamedParameterJdbcTemplate namedParameterJdbcTemplate;
public BatchSaveUtils(NamedParameterJdbcTemplate namedParameterJdbcTemplate) {
BatchSaveUtils.namedParameterJdbcTemplate = namedParameterJdbcTemplate;
}
public static <T> void batchSave(List<T> dtos) {
if (CollectionUtils.isEmpty(dtos)) {
return;
}
// 表名
String table = CaseFormat.LOWER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, dtos.get(0).getClass().getSimpleName().toLowerCase());
BeanPropertySqlParameterSource[] args = dtos.stream()
.map(BeanPropertySqlParameterSource::new)
.toArray(BeanPropertySqlParameterSource[]::new);
List<String> fields = Arrays.stream(FieldUtils.getAllFields(User.class))
.map(Field::getName)
.filter(name -> !"serialVersionUID".equalsIgnoreCase(name))
.toList();
// 列名
String columns = StringUtils.join(fields.stream().map(name -> CaseFormat.LOWER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, name)).toList(), ",");
// 值
String values = StringUtils.join(fields, ",:");
String sql = "INSERT INTO " + table + "(" + columns + ") VALUES (:" + values + ")";
namedParameterJdbcTemplate.batchUpdate(sql, args);
}
}

123
backend/services/system-setting/src/test/java/io/metersphere/system/controller/UserControllerTests.java
View File

@ -2,6 +2,7 @@ package io.metersphere.system.controller;
import io.metersphere.sdk.dto.UserDTO;
import io.metersphere.sdk.util.JSON;
import io.metersphere.system.domain.User;
import jakarta.annotation.Resource;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
@ -13,6 +14,8 @@ import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import java.util.ArrayList;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
@ -23,24 +26,6 @@ public class UserControllerTests {
@Resource
private MockMvc mockMvc;
@Test
@Order(4)
public void testSelectAll() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.get("/user/list-all"))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(jsonPath("$.data[0].name").value("admin"));
}
@Test
@Order(3)
public void testGetUser() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.get("/user/get/admin"))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(jsonPath("$.data.id").value("admin"));
}
@Test
@Order(1)
public void testAddUser() throws Exception {
@ -79,4 +64,106 @@ public class UserControllerTests {
.andExpect(status().isBadRequest())
.andExpect(content().contentType(MediaType.APPLICATION_JSON));
}
@Test
@Order(3)
public void testGetUser() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.get("/user/get/admin"))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(jsonPath("$.data.id").value("admin"));
}
@Test
@Order(4)
public void testSelectAll() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.get("/user/list-all"))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(jsonPath("$.data[0].name").value("admin"));
}
@Test
@Order(5)
public void testBatchAddUser() throws Exception {
var users = new ArrayList<User>();
for (int i = 0; i < 1000; i++) {
User user = new User();
user.setId("batch1_" + i);
user.setName("batch1_" + i);
user.setCreateUser("system");
user.setSource("LOCAL");
user.setEmail("bin@fit2cloud.com");
user.setStatus("enabled");
user.setCreateTime(System.currentTimeMillis());
user.setUpdateTime(System.currentTimeMillis());
users.add(user);
}
mockMvc.perform(MockMvcRequestBuilders.post("/user/batch-add")
.content(JSON.toJSONString(users))
.contentType(MediaType.APPLICATION_JSON))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON));
}
@Test
@Order(6)
public void testBatchAddUser2() throws Exception {
var users = new ArrayList<User>();
for (int i = 0; i < 1000; i++) {
User user = new User();
user.setId("batch2_" + i);
user.setName("batch2_" + i);
user.setCreateUser("system");
user.setSource("LOCAL");
user.setEmail("bin@fit2cloud.com");
user.setStatus("enabled");
user.setCreateTime(System.currentTimeMillis());
user.setUpdateTime(System.currentTimeMillis());
users.add(user);
}
mockMvc.perform(MockMvcRequestBuilders.post("/user/batch-add2")
.content(JSON.toJSONString(users))
.contentType(MediaType.APPLICATION_JSON))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON));
}
@Test
@Order(7)
public void testBatchAddUser3() throws Exception {
var users = new ArrayList<User>();
for (int i = 0; i < 1000; i++) {
User user = new User();
user.setId("batch3_" + i);
user.setName("batch3_" + i);
user.setCreateUser("system");
user.setSource("LOCAL");
user.setEmail("bin@fit2cloud.com");
user.setStatus("enabled");
user.setCreateTime(System.currentTimeMillis());
user.setUpdateTime(System.currentTimeMillis());
users.add(user);
}
mockMvc.perform(MockMvcRequestBuilders.post("/user/batch-add3")
.content(JSON.toJSONString(users))
.contentType(MediaType.APPLICATION_JSON))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON));
}
@Test
@Order(8)
public void testCount() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.get("/user/count")
.contentType(MediaType.APPLICATION_JSON))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andExpect(jsonPath("$.data").value(3001));
}
}

2
backend/services/system-setting/src/test/resources/application.properties
View File

@ -13,7 +13,7 @@ quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock=true
#
logging.file.path=/opt/metersphere/logs/metersphere
# Hikari
spring.datasource.url=jdbc:mysql://${embedded.mysql.host}:${embedded.mysql.port}/test?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2B8
spring.datasource.url=jdbc:mysql://${embedded.mysql.host}:${embedded.mysql.port}/test?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true&useSSL=false&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION%27
spring.datasource.username=${embedded.mysql.user}
spring.datasource.password=${embedded.mysql.password}
spring.datasource.type=com.zaxxer.hikari.HikariDataSource