From e8233fe33676da36a8f8fb666aeb52592e996a2a Mon Sep 17 00:00:00 2001 From: WangXu10 Date: Mon, 29 Jan 2024 11:43:59 +0800 Subject: [PATCH] =?UTF-8?q?refactor(=E7=B3=BB=E7=BB=9F):=20=E6=B8=85?= =?UTF-8?q?=E7=90=86=E6=8A=A5=E5=91=8A=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=89=80=E6=9C=89=E9=A1=B9=E7=9B=AE=E5=85=B1=E7=94=A8=E4=B8=80?= =?UTF-8?q?=E4=B8=AA&=E9=A1=B9=E7=9B=AE=E5=90=AF=E5=8A=A8=E6=97=B6?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../listener/AppStartListener.java | 7 ++ .../migration/3.0.0/dml/V3.0.0_11_1__data.sql | 2 - .../project/job/CleanUpReportJob.java | 6 + .../CreateApplicationResourceService.java | 8 +- .../service/ProjectApplicationService.java | 11 +- .../CreateApplicationResourceTests.java | 2 + .../system/job/CleanUpReportJob.java | 112 ++++++++++++++++++ .../system/schedule/ScheduleManager.java | 5 +- .../system/service/BaseCleanUpReport.java | 8 ++ .../system/service/BaseScheduleService.java | 55 +++++++++ .../controller/CleanReportJobTests.java | 33 ++++++ 11 files changed, 237 insertions(+), 12 deletions(-) create mode 100644 backend/services/system-setting/src/main/java/io/metersphere/system/job/CleanUpReportJob.java create mode 100644 backend/services/system-setting/src/main/java/io/metersphere/system/service/BaseCleanUpReport.java create mode 100644 backend/services/system-setting/src/main/java/io/metersphere/system/service/BaseScheduleService.java create mode 100644 backend/services/system-setting/src/test/java/io/metersphere/system/controller/CleanReportJobTests.java diff --git a/backend/app/src/main/java/io/metersphere/listener/AppStartListener.java b/backend/app/src/main/java/io/metersphere/listener/AppStartListener.java index d7519a49ae..8013056a12 100644 --- a/backend/app/src/main/java/io/metersphere/listener/AppStartListener.java +++ b/backend/app/src/main/java/io/metersphere/listener/AppStartListener.java @@ -7,6 +7,7 @@ import io.metersphere.sdk.file.FileCenter; import io.metersphere.sdk.file.MinioRepository; import io.metersphere.sdk.util.CommonBeanFactory; import io.metersphere.sdk.util.LogUtils; +import io.metersphere.system.service.BaseScheduleService; import io.metersphere.system.service.PluginLoadService; import io.minio.MinioClient; import jakarta.annotation.Resource; @@ -22,12 +23,18 @@ public class AppStartListener implements ApplicationRunner { @Resource private MinioClient minioClient; + @Resource + private BaseScheduleService baseScheduleService; + @Override public void run(ApplicationArguments args) throws Exception { LogUtils.info("================= 应用启动 ================="); // 初始化MinIO配置 ((MinioRepository) FileCenter.getRepository(StorageType.MINIO)).init(minioClient); + LogUtils.info("初始化定时任务"); + baseScheduleService.startEnableSchedules(); + // 注册所有监听源 LogUtils.info("初始化接口事件源"); ApiEventSource apiEventSource = CommonBeanFactory.getBean(ApiEventSource.class); diff --git a/backend/framework/domain/src/main/resources/migration/3.0.0/dml/V3.0.0_11_1__data.sql b/backend/framework/domain/src/main/resources/migration/3.0.0/dml/V3.0.0_11_1__data.sql index d9d0d30db7..5e8cb03d8c 100644 --- a/backend/framework/domain/src/main/resources/migration/3.0.0/dml/V3.0.0_11_1__data.sql +++ b/backend/framework/domain/src/main/resources/migration/3.0.0/dml/V3.0.0_11_1__data.sql @@ -705,8 +705,6 @@ INSERT INTO message_task_blob(id, template) VALUES (@bug_comment_reply_id, 'mess --- 初始化定时任务数据 -INSERT INTO schedule(`id`, `key`, `type`,`resource_type`, `value`, `job`, `enable`, `resource_id`, `create_user`, `create_time`, `update_time`, `project_id`, `name`, `config`) VALUES (UUID_SHORT(), '100001100001', 'CRON','CLEAN_REPORT', '0 0 2 * * ?', 'io.metersphere.project.job.CleanUpReportJob', true, '100001100001', 'admin', unix_timestamp() * 1000, unix_timestamp() * 1000, '100001100001', 'Clean Report Job', NULL); -- 初始化默认项目版本配置项 INSERT INTO project_application (`project_id`, `type`, `type_value`) VALUES ('100001100001', 'VERSION_ENABLE', 'FALSE'); diff --git a/backend/services/project-management/src/main/java/io/metersphere/project/job/CleanUpReportJob.java b/backend/services/project-management/src/main/java/io/metersphere/project/job/CleanUpReportJob.java index b401a767cc..9ef7d1e070 100644 --- a/backend/services/project-management/src/main/java/io/metersphere/project/job/CleanUpReportJob.java +++ b/backend/services/project-management/src/main/java/io/metersphere/project/job/CleanUpReportJob.java @@ -1,3 +1,7 @@ + +//TODO 后续清理 （合并完没问题再清理）=====清理报告定时任务只有一个，cron表达式凌晨定时扫描 + +/* package io.metersphere.project.job; import io.metersphere.system.schedule.BaseScheduleJob; @@ -23,3 +27,5 @@ public class CleanUpReportJob extends BaseScheduleJob { return new TriggerKey(projectId, CleanUpReportJob.class.getName()); } } +*/ + diff --git a/backend/services/project-management/src/main/java/io/metersphere/project/service/CreateApplicationResourceService.java b/backend/services/project-management/src/main/java/io/metersphere/project/service/CreateApplicationResourceService.java index 90f2329563..805f1a16c6 100644 --- a/backend/services/project-management/src/main/java/io/metersphere/project/service/CreateApplicationResourceService.java +++ b/backend/services/project-management/src/main/java/io/metersphere/project/service/CreateApplicationResourceService.java @@ -1,3 +1,6 @@ + +//TODO 后续清理（合并完没问题再清理）=====清理报告定时任务只有一个，cron表达式凌晨定时扫描 创建项目不再创建定时任务 +/* package io.metersphere.project.service; import io.metersphere.project.job.CleanUpReportJob; @@ -9,9 +12,11 @@ import io.metersphere.system.service.CreateProjectResourceService; import jakarta.annotation.Resource; import org.springframework.stereotype.Component; +*/ /** * @author wx - */ + *//* + @Component public class CreateApplicationResourceService implements CreateProjectResourceService { @@ -41,3 +46,4 @@ public class CreateApplicationResourceService implements CreateProjectResourceSe CleanUpReportJob.class); } } +*/ diff --git a/backend/services/project-management/src/main/java/io/metersphere/project/service/ProjectApplicationService.java b/backend/services/project-management/src/main/java/io/metersphere/project/service/ProjectApplicationService.java index fb9dcbb23d..688377c2be 100644 --- a/backend/services/project-management/src/main/java/io/metersphere/project/service/ProjectApplicationService.java +++ b/backend/services/project-management/src/main/java/io/metersphere/project/service/ProjectApplicationService.java @@ -9,7 +9,6 @@ import io.metersphere.project.domain.ProjectApplication; import io.metersphere.project.domain.ProjectApplicationExample; import io.metersphere.project.dto.ModuleDTO; import io.metersphere.project.job.BugSyncJob; -import io.metersphere.project.job.CleanUpReportJob; import io.metersphere.project.mapper.*; import io.metersphere.project.request.ProjectApplicationRequest; import io.metersphere.project.utils.ModuleSortUtils; @@ -34,7 +33,6 @@ import io.metersphere.system.service.ServiceIntegrationService; import io.metersphere.system.utils.ServiceUtils; import jakarta.annotation.Resource; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.pf4j.PluginWrapper; import org.springframework.stereotype.Service; @@ -107,17 +105,18 @@ public class ProjectApplicationService { } private void doBeforeUpdate(ProjectApplication application, String currentUser) { - String type = application.getType(); + //TODO 后续清理（合并完没问题再清理）===== 清理报告只有一个定时任务，项目配置时不需要在添加定时任务了 + /*String type = application.getType(); if (StringUtils.equals(type, ProjectApplicationType.TEST_PLAN.TEST_PLAN_CLEAN_REPORT.name()) || StringUtils.equals(type, ProjectApplicationType.UI.UI_CLEAN_REPORT.name()) || StringUtils.equals(type, ProjectApplicationType.LOAD_TEST.LOAD_TEST_CLEAN_REPORT.name()) || StringUtils.equals(type, ProjectApplicationType.API.API_CLEAN_REPORT.name())) { //清除 测试计划/UI测试/性能测试/接口测试 报告 定时任务 this.doHandleSchedule(application, currentUser); - } + }*/ } - private void doHandleSchedule(ProjectApplication application, String currentUser) { + /*private void doHandleSchedule(ProjectApplication application, String currentUser) { String typeValue = application.getTypeValue(); String projectId = application.getProjectId(); Boolean enable = BooleanUtils.isTrue(Boolean.valueOf(typeValue)); @@ -151,7 +150,7 @@ public class ProjectApplicationService { CleanUpReportJob.class); }); - } + }*/ /** diff --git a/backend/services/project-management/src/test/java/io/metersphere/project/controller/CreateApplicationResourceTests.java b/backend/services/project-management/src/test/java/io/metersphere/project/controller/CreateApplicationResourceTests.java index 27c6440b79..00086bc26f 100644 --- a/backend/services/project-management/src/test/java/io/metersphere/project/controller/CreateApplicationResourceTests.java +++ b/backend/services/project-management/src/test/java/io/metersphere/project/controller/CreateApplicationResourceTests.java @@ -1,3 +1,4 @@ +/* package io.metersphere.project.controller; import io.metersphere.project.service.CreateApplicationResourceService; @@ -23,3 +24,4 @@ public class CreateApplicationResourceTests { resourceService.createResources("test_project_id"); } } +*/ diff --git a/backend/services/system-setting/src/main/java/io/metersphere/system/job/CleanUpReportJob.java b/backend/services/system-setting/src/main/java/io/metersphere/system/job/CleanUpReportJob.java new file mode 100644 index 0000000000..9297635d1f --- /dev/null +++ b/backend/services/system-setting/src/main/java/io/metersphere/system/job/CleanUpReportJob.java @@ -0,0 +1,112 @@ +package io.metersphere.system.job; + +import com.fit2cloud.quartz.anno.QuartzScheduled; +import io.metersphere.project.domain.Project; +import io.metersphere.project.domain.ProjectApplication; +import io.metersphere.project.domain.ProjectApplicationExample; +import io.metersphere.project.domain.ProjectExample; +import io.metersphere.project.mapper.ProjectApplicationMapper; +import io.metersphere.project.mapper.ProjectMapper; +import io.metersphere.sdk.constants.ProjectApplicationType; +import io.metersphere.system.service.BaseCleanUpReport; +import jakarta.annotation.Resource; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/** + * @author wx + */ +@Component +public class CleanUpReportJob { + + @Resource + private ProjectMapper projectMapper; + @Resource + private ProjectApplicationMapper projectApplicationMapper; + + @Autowired + private ApplicationContext applicationContext; + + /** + * 清理报告定时任务（所有项目共用一个） + */ + @QuartzScheduled(cron = "0 0 1 * * ?") + public void cleanReport() { + + long count = getProjectCount(); + + long pages = Double.valueOf(Math.ceil(count / 100.0)).longValue(); + + for (int i = 1; i <= pages; i++) { + Thread.startVirtualThread(new Runnable() { + @Override + public void run() { + ProjectExample example = new ProjectExample(); + example.createCriteria().andDeletedEqualTo(false); + List projects = projectMapper.selectByExample(example); + projects.forEach(project -> { + ProjectApplicationExample applicationExample = new ProjectApplicationExample(); + //test_plan + applicationExample.createCriteria().andProjectIdEqualTo(project.getId()).andTypeEqualTo(ProjectApplicationType.TEST_PLAN.TEST_PLAN_CLEAN_REPORT.name()); + List testPlan = projectApplicationMapper.selectByExample(applicationExample); + Map map = new HashMap<>(); + if (CollectionUtils.isNotEmpty(testPlan)) { + map.put(ProjectApplicationType.TEST_PLAN.TEST_PLAN_CLEAN_REPORT.name(), testPlan.get(0).getTypeValue()); + } else { + map.put(ProjectApplicationType.TEST_PLAN.TEST_PLAN_CLEAN_REPORT.name(), "3M"); + } + + //ui + applicationExample.clear(); + applicationExample.createCriteria().andProjectIdEqualTo(project.getId()).andTypeEqualTo(ProjectApplicationType.UI.UI_CLEAN_REPORT.name()); + List ui = projectApplicationMapper.selectByExample(applicationExample); + if (CollectionUtils.isNotEmpty(ui)) { + map.put(ProjectApplicationType.UI.UI_CLEAN_REPORT.name(), ui.get(0).getTypeValue()); + } else { + map.put(ProjectApplicationType.UI.UI_CLEAN_REPORT.name(), "3M"); + } + + //load_test + applicationExample.clear(); + applicationExample.createCriteria().andProjectIdEqualTo(project.getId()).andTypeEqualTo(ProjectApplicationType.LOAD_TEST.LOAD_TEST_CLEAN_REPORT.name()); + List loadTest = projectApplicationMapper.selectByExample(applicationExample); + if (CollectionUtils.isNotEmpty(loadTest)) { + map.put(ProjectApplicationType.LOAD_TEST.LOAD_TEST_CLEAN_REPORT.name(), loadTest.get(0).getTypeValue()); + } else { + map.put(ProjectApplicationType.LOAD_TEST.LOAD_TEST_CLEAN_REPORT.name(), "3M"); + } + + //api + applicationExample.clear(); + applicationExample.createCriteria().andProjectIdEqualTo(project.getId()).andTypeEqualTo(ProjectApplicationType.API.API_CLEAN_REPORT.name()); + List api = projectApplicationMapper.selectByExample(applicationExample); + if (CollectionUtils.isNotEmpty(api)) { + map.put(ProjectApplicationType.API.API_CLEAN_REPORT.name(), api.get(0).getTypeValue()); + } else { + map.put(ProjectApplicationType.API.API_CLEAN_REPORT.name(), "3M"); + } + + Map beansOfType = applicationContext.getBeansOfType(BaseCleanUpReport.class); + beansOfType.forEach((k, v) -> { + v.cleanReport(map, project.getId()); + }); + + }); + } + }); + } + } + + private long getProjectCount() { + ProjectExample example = new ProjectExample(); + example.createCriteria().andDeletedEqualTo(false); + return projectMapper.countByExample(example); + } +} diff --git a/backend/services/system-setting/src/main/java/io/metersphere/system/schedule/ScheduleManager.java b/backend/services/system-setting/src/main/java/io/metersphere/system/schedule/ScheduleManager.java index 4649e3c59e..3b07872564 100644 --- a/backend/services/system-setting/src/main/java/io/metersphere/system/schedule/ScheduleManager.java +++ b/backend/services/system-setting/src/main/java/io/metersphere/system/schedule/ScheduleManager.java @@ -127,9 +127,8 @@ public class ScheduleManager { /** * 添加或修改 cronJob */ - public void addOrUpdateCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, JobDataMap jobDataMap) + public void addOrUpdateCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, JobDataMap jobDataMap) throws SchedulerException { - LogUtils.info("AddOrUpdateCronJob: " + jobKey.getName() + "," + triggerKey.getGroup()); if (scheduler.checkExists(triggerKey)) { @@ -139,7 +138,7 @@ public class ScheduleManager { } } - public void addOrUpdateCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron) throws SchedulerException { + public void addOrUpdateCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron) throws SchedulerException { addOrUpdateCronJob(jobKey, triggerKey, jobClass, cron, null); } diff --git a/backend/services/system-setting/src/main/java/io/metersphere/system/service/BaseCleanUpReport.java b/backend/services/system-setting/src/main/java/io/metersphere/system/service/BaseCleanUpReport.java new file mode 100644 index 0000000000..d828b033c4 --- /dev/null +++ b/backend/services/system-setting/src/main/java/io/metersphere/system/service/BaseCleanUpReport.java @@ -0,0 +1,8 @@ +package io.metersphere.system.service; + + +import java.util.Map; + +public interface BaseCleanUpReport { + void cleanReport(Map map, String projectId); +} diff --git a/backend/services/system-setting/src/main/java/io/metersphere/system/service/BaseScheduleService.java b/backend/services/system-setting/src/main/java/io/metersphere/system/service/BaseScheduleService.java new file mode 100644 index 0000000000..512f09a8df --- /dev/null +++ b/backend/services/system-setting/src/main/java/io/metersphere/system/service/BaseScheduleService.java @@ -0,0 +1,55 @@ +package io.metersphere.system.service; + +import io.metersphere.sdk.util.JSON; +import io.metersphere.sdk.util.LogUtils; +import io.metersphere.system.domain.Schedule; +import io.metersphere.system.domain.ScheduleExample; +import io.metersphere.system.mapper.ScheduleMapper; +import io.metersphere.system.schedule.ScheduleManager; +import jakarta.annotation.Resource; +import org.quartz.JobKey; +import org.quartz.TriggerKey; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +@Service +@Transactional(rollbackFor = Exception.class) +public class BaseScheduleService { + + @Resource + private ScheduleMapper scheduleMapper; + @Resource + private ScheduleManager scheduleManager; + + public void startEnableSchedules() { + List Schedules = getSchedule(); + Schedules.forEach(schedule -> { + try { + if (schedule.getEnable()) { + LogUtils.info("初始化任务：" + JSON.toJSONString(schedule)); + scheduleManager.addOrUpdateCronJob(new JobKey(schedule.getKey()), + new TriggerKey(schedule.getKey()), Class.forName(schedule.getJob()), schedule.getValue(), + scheduleManager.getDefaultJobDataMap(schedule, schedule.getValue(), schedule.getCreateUser())); + } else { + // 删除关闭的job + removeJob(schedule.getKey()); + } + } catch (Exception e) { + LogUtils.error("初始化任务失败", e); + } + }); + + } + + private List getSchedule() { + ScheduleExample example = new ScheduleExample(); + example.createCriteria(); + return scheduleMapper.selectByExample(example); + } + + private void removeJob(String key) { + scheduleManager.removeJob(new JobKey(key), new TriggerKey(key)); + } +} diff --git a/backend/services/system-setting/src/test/java/io/metersphere/system/controller/CleanReportJobTests.java b/backend/services/system-setting/src/test/java/io/metersphere/system/controller/CleanReportJobTests.java new file mode 100644 index 0000000000..12105b6f10 --- /dev/null +++ b/backend/services/system-setting/src/test/java/io/metersphere/system/controller/CleanReportJobTests.java @@ -0,0 +1,33 @@ +package io.metersphere.system.controller; + +import io.metersphere.system.base.BaseTest; +import io.metersphere.system.job.CleanUpReportJob; +import jakarta.annotation.Resource; +import org.apache.commons.lang3.BooleanUtils; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlConfig; + +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@AutoConfigureMockMvc +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class CleanReportJobTests extends BaseTest { + + @Resource + private CleanUpReportJob cleanUpReportJob; + + + @Test + @Order(1) + @Sql(scripts = {"/dml/init_project.sql"}, config = @SqlConfig(encoding = "utf-8", transactionMode = SqlConfig.TransactionMode.ISOLATED)) + public void test() throws Exception { + BooleanUtils.isTrue(null); + cleanUpReportJob.cleanReport(); + } + +}