From bf2b874fb59003dbc60082f18776e75b0d8a9cc7 Mon Sep 17 00:00:00 2001 From: CaptainB Date: Wed, 24 May 2023 12:17:17 +0800 Subject: [PATCH] =?UTF-8?q?build:=20=E6=89=B9=E9=87=8F=E6=8F=92=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/metersphere/sdk/mapper/UserMapper.java | 5 + .../io/metersphere/sdk/mapper/UserMapper.xml | 17 +++ .../system/controller/UserController.java | 21 +++ .../system/service/UserService.java | 62 ++++++++- .../system/util/BatchSaveUtils.java | 48 +++++++ .../controller/UserControllerTests.java | 123 +++++++++++++++--- .../src/test/resources/application.properties | 2 +- 7 files changed, 257 insertions(+), 21 deletions(-) create mode 100644 backend/services/system-setting/src/main/java/io/metersphere/system/util/BatchSaveUtils.java diff --git a/backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.java b/backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.java index 63c75beb79..f2b59777ec 100644 --- a/backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.java +++ b/backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.java @@ -2,6 +2,7 @@ package io.metersphere.sdk.mapper; import io.metersphere.sdk.dto.UserDTO; import io.metersphere.system.domain.User; +import org.apache.ibatis.annotations.Param; import java.util.List; @@ -9,4 +10,8 @@ public interface UserMapper { UserDTO selectById(String id); List findAll(); + + void insert(User user); + + void batchSave(@Param("users") List users); } diff --git a/backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.xml b/backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.xml index 4ae32045f4..dbec8c27c2 100644 --- a/backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.xml +++ b/backend/framework/sdk/src/main/java/io/metersphere/sdk/mapper/UserMapper.xml @@ -13,4 +13,21 @@ SELECT * FROM user + + + INSERT INTO user(id, name, email, password, status, create_time, update_time, language, last_workspace_id, phone, + source, last_project_id, create_user) + VALUES (#{id}, #{name}, #{email}, #{password}, #{status}, #{createTime}, #{updateTime}, #{language}, + #{lastWorkspaceId}, #{phone}, #{source}, #{lastProjectId}, #{createUser}) + + + + INSERT INTO user(id, name, email, password, status, create_time, update_time, language, last_workspace_id, phone, + source, last_project_id, create_user) + VALUES + + (#{user.id}, #{user.name}, #{user.email}, #{user.password}, #{user.status}, #{user.createTime}, #{user.updateTime}, #{user.language}, + #{user.lastWorkspaceId}, #{user.phone}, #{user.source}, #{user.lastProjectId}, #{user.createUser}) + + \ No newline at end of file diff --git a/backend/services/system-setting/src/main/java/io/metersphere/system/controller/UserController.java b/backend/services/system-setting/src/main/java/io/metersphere/system/controller/UserController.java index 256bcd6fa1..2e00567fee 100644 --- a/backend/services/system-setting/src/main/java/io/metersphere/system/controller/UserController.java +++ b/backend/services/system-setting/src/main/java/io/metersphere/system/controller/UserController.java @@ -31,4 +31,25 @@ public class UserController { public boolean addUser(@Validated({Created.class}) @RequestBody UserDTO user) { return userService.save(user); } + + @PostMapping("/batch-add") + public boolean batchSaveUser(@Validated({Created.class}) @RequestBody List user) { + return userService.batchSave(user); + } + + @PostMapping("/batch-add2") + public boolean batchSaveUser2(@Validated({Created.class}) @RequestBody List user) { + return userService.batchSave2(user); + } + + @PostMapping("/batch-add3") + public boolean batchSaveUser3(@Validated({Created.class}) @RequestBody List user) { + return userService.batchSave3(user); + } + + @GetMapping("/count") + public long batchSaveUser() { + return userService.count(); + } + } diff --git a/backend/services/system-setting/src/main/java/io/metersphere/system/service/UserService.java b/backend/services/system-setting/src/main/java/io/metersphere/system/service/UserService.java index 2dec1707e3..873dd26a9e 100644 --- a/backend/services/system-setting/src/main/java/io/metersphere/system/service/UserService.java +++ b/backend/services/system-setting/src/main/java/io/metersphere/system/service/UserService.java @@ -5,9 +5,11 @@ import io.metersphere.sdk.mapper.UserMapper; import io.metersphere.sdk.util.BeanUtils; import io.metersphere.system.domain.User; import io.metersphere.system.domain.UserExtend; +import io.metersphere.system.util.BatchSaveUtils; import jakarta.annotation.Resource; +import org.apache.ibatis.session.SqlSession; +import org.apache.ibatis.session.SqlSessionFactory; import org.springframework.data.jdbc.core.JdbcAggregateTemplate; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -21,7 +23,7 @@ public class UserService { @Resource private JdbcAggregateTemplate jdbcAggregateTemplate; @Resource - private JdbcTemplate jdbcTemplate; + private SqlSessionFactory sqlSessionFactory; public boolean save(UserDTO entity) { User user = new User(); @@ -41,4 +43,60 @@ public class UserService { public List list() { return userMapper.findAll(); } + + public boolean batchSave(List users) { + long start = System.currentTimeMillis(); + BatchSaveUtils.batchSave(users); + System.out.println("batch save cost: " + (System.currentTimeMillis() - start) + "ms"); + return true; + } + + public boolean batchSave2(List users) { + long start = System.currentTimeMillis(); + + try (SqlSession sqlSession = sqlSessionFactory.openSession()) { + UserMapper mapper = sqlSession.getMapper(UserMapper.class); + for (int i = 0, size = users.size(); i < size; i++) { + mapper.insert(users.get(i)); + if (i % 100 == 0) { + sqlSession.flushStatements(); + } + } + sqlSession.flushStatements(); + } + System.out.println("batch save cost: " + (System.currentTimeMillis() - start) + "ms"); + return true; + } + + public boolean batchSave3(List users) { + + long start = System.currentTimeMillis(); + int batchSize = 100; + int size = users.size(); + int pageSize = size / batchSize; + if (pageSize == 0) { + userMapper.batchSave(users); + System.out.println("batch save cost: " + (System.currentTimeMillis() - start) + "ms"); + return true; + } + + for (int i = 0; i < pageSize; i++) { + int startIndex = i * batchSize; + List sub = users.subList(startIndex, startIndex + batchSize); + userMapper.batchSave(sub); + } + + if (size % batchSize != 0) { + int startIndex = pageSize * batchSize; + List sub = users.subList(startIndex, size); + userMapper.batchSave(sub); + } + System.out.println("batch save cost: " + (System.currentTimeMillis() - start) + "ms"); + return true; + } + + public long count() { + return jdbcAggregateTemplate.count(User.class); + } + } diff --git a/backend/services/system-setting/src/main/java/io/metersphere/system/util/BatchSaveUtils.java b/backend/services/system-setting/src/main/java/io/metersphere/system/util/BatchSaveUtils.java new file mode 100644 index 0000000000..f3a9c42d9b --- /dev/null +++ b/backend/services/system-setting/src/main/java/io/metersphere/system/util/BatchSaveUtils.java @@ -0,0 +1,48 @@ +package io.metersphere.system.util; + +import com.google.common.base.CaseFormat; +import io.metersphere.system.domain.User; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.springframework.jdbc.core.namedparam.BeanPropertySqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Component; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.List; + +@Component +public class BatchSaveUtils { + private static NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + public BatchSaveUtils(NamedParameterJdbcTemplate namedParameterJdbcTemplate) { + BatchSaveUtils.namedParameterJdbcTemplate = namedParameterJdbcTemplate; + } + + + public static void batchSave(List dtos) { + if (CollectionUtils.isEmpty(dtos)) { + return; + } + // 表名 + String table = CaseFormat.LOWER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, dtos.get(0).getClass().getSimpleName().toLowerCase()); + + BeanPropertySqlParameterSource[] args = dtos.stream() + .map(BeanPropertySqlParameterSource::new) + .toArray(BeanPropertySqlParameterSource[]::new); + + List fields = Arrays.stream(FieldUtils.getAllFields(User.class)) + .map(Field::getName) + .filter(name -> !"serialVersionUID".equalsIgnoreCase(name)) + .toList(); + // 列名 + String columns = StringUtils.join(fields.stream().map(name -> CaseFormat.LOWER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, name)).toList(), ","); + // 值 + String values = StringUtils.join(fields, ",:"); + + String sql = "INSERT INTO " + table + "(" + columns + ") VALUES (:" + values + ")"; + namedParameterJdbcTemplate.batchUpdate(sql, args); + } +} diff --git a/backend/services/system-setting/src/test/java/io/metersphere/system/controller/UserControllerTests.java b/backend/services/system-setting/src/test/java/io/metersphere/system/controller/UserControllerTests.java index 0072df43a9..ac05db9d6e 100644 --- a/backend/services/system-setting/src/test/java/io/metersphere/system/controller/UserControllerTests.java +++ b/backend/services/system-setting/src/test/java/io/metersphere/system/controller/UserControllerTests.java @@ -2,6 +2,7 @@ package io.metersphere.system.controller; import io.metersphere.sdk.dto.UserDTO; import io.metersphere.sdk.util.JSON; +import io.metersphere.system.domain.User; import jakarta.annotation.Resource; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; @@ -13,6 +14,8 @@ import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import java.util.ArrayList; + import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; @@ -23,24 +26,6 @@ public class UserControllerTests { @Resource private MockMvc mockMvc; - @Test - @Order(4) - public void testSelectAll() throws Exception { - mockMvc.perform(MockMvcRequestBuilders.get("/user/list-all")) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andExpect(jsonPath("$.data[0].name").value("admin")); - } - - @Test - @Order(3) - public void testGetUser() throws Exception { - mockMvc.perform(MockMvcRequestBuilders.get("/user/get/admin")) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andExpect(jsonPath("$.data.id").value("admin")); - } - @Test @Order(1) public void testAddUser() throws Exception { @@ -79,4 +64,106 @@ public class UserControllerTests { .andExpect(status().isBadRequest()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)); } + + @Test + @Order(3) + public void testGetUser() throws Exception { + mockMvc.perform(MockMvcRequestBuilders.get("/user/get/admin")) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.data.id").value("admin")); + } + + @Test + @Order(4) + public void testSelectAll() throws Exception { + mockMvc.perform(MockMvcRequestBuilders.get("/user/list-all")) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.data[0].name").value("admin")); + } + + + @Test + @Order(5) + public void testBatchAddUser() throws Exception { + var users = new ArrayList(); + for (int i = 0; i < 1000; i++) { + User user = new User(); + user.setId("batch1_" + i); + user.setName("batch1_" + i); + user.setCreateUser("system"); + user.setSource("LOCAL"); + user.setEmail("bin@fit2cloud.com"); + user.setStatus("enabled"); + user.setCreateTime(System.currentTimeMillis()); + user.setUpdateTime(System.currentTimeMillis()); + users.add(user); + } + + mockMvc.perform(MockMvcRequestBuilders.post("/user/batch-add") + .content(JSON.toJSONString(users)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)); + } + + @Test + @Order(6) + public void testBatchAddUser2() throws Exception { + var users = new ArrayList(); + for (int i = 0; i < 1000; i++) { + User user = new User(); + user.setId("batch2_" + i); + user.setName("batch2_" + i); + user.setCreateUser("system"); + user.setSource("LOCAL"); + user.setEmail("bin@fit2cloud.com"); + user.setStatus("enabled"); + user.setCreateTime(System.currentTimeMillis()); + user.setUpdateTime(System.currentTimeMillis()); + users.add(user); + } + + mockMvc.perform(MockMvcRequestBuilders.post("/user/batch-add2") + .content(JSON.toJSONString(users)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)); + } + + @Test + @Order(7) + public void testBatchAddUser3() throws Exception { + var users = new ArrayList(); + for (int i = 0; i < 1000; i++) { + User user = new User(); + user.setId("batch3_" + i); + user.setName("batch3_" + i); + user.setCreateUser("system"); + user.setSource("LOCAL"); + user.setEmail("bin@fit2cloud.com"); + user.setStatus("enabled"); + user.setCreateTime(System.currentTimeMillis()); + user.setUpdateTime(System.currentTimeMillis()); + users.add(user); + } + + mockMvc.perform(MockMvcRequestBuilders.post("/user/batch-add3") + .content(JSON.toJSONString(users)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)); + } + + @Test + @Order(8) + public void testCount() throws Exception { + mockMvc.perform(MockMvcRequestBuilders.get("/user/count") + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.data").value(3001)); + } + } diff --git a/backend/services/system-setting/src/test/resources/application.properties b/backend/services/system-setting/src/test/resources/application.properties index 1b9a09bfff..71035947a4 100644 --- a/backend/services/system-setting/src/test/resources/application.properties +++ b/backend/services/system-setting/src/test/resources/application.properties @@ -13,7 +13,7 @@ quartz.properties.org.quartz.jobStore.acquireTriggersWithinLock=true # logging.file.path=/opt/metersphere/logs/metersphere # Hikari -spring.datasource.url=jdbc:mysql://${embedded.mysql.host}:${embedded.mysql.port}/test?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2B8 +spring.datasource.url=jdbc:mysql://${embedded.mysql.host}:${embedded.mysql.port}/test?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true&useSSL=false&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION%27 spring.datasource.username=${embedded.mysql.user} spring.datasource.password=${embedded.mysql.password} spring.datasource.type=com.zaxxer.hikari.HikariDataSource