diff --git a/backend/pom.xml b/backend/pom.xml index e9c54878a5..4fe53a3fb0 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -17,6 +17,7 @@ UTF-8 1.4.0 1.8 + 5.2.1 @@ -137,6 +138,19 @@ slf4j-simple + + + org.apache.jmeter + ApacheJMeter_http + ${jmeter.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + diff --git a/backend/src/main/java/io/metersphere/runner/Engine.java b/backend/src/main/java/io/metersphere/runner/Engine.java new file mode 100644 index 0000000000..a93023b6f1 --- /dev/null +++ b/backend/src/main/java/io/metersphere/runner/Engine.java @@ -0,0 +1,9 @@ +package io.metersphere.runner; + +public interface Engine { + boolean init(EngineContext context); + + void start(); + + void stop(); +} diff --git a/backend/src/main/java/io/metersphere/runner/EngineContext.java b/backend/src/main/java/io/metersphere/runner/EngineContext.java new file mode 100644 index 0000000000..b6b5e890fe --- /dev/null +++ b/backend/src/main/java/io/metersphere/runner/EngineContext.java @@ -0,0 +1,24 @@ +package io.metersphere.runner; + +import java.io.InputStream; + +public class EngineContext { + private String engineId; + private InputStream inputStream; + + public String getEngineId() { + return engineId; + } + + public void setEngineId(String engineId) { + this.engineId = engineId; + } + + public InputStream getInputStream() { + return inputStream; + } + + public void setInputStream(InputStream inputStream) { + this.inputStream = inputStream; + } +} diff --git a/backend/src/main/java/io/metersphere/runner/EngineFactory.java b/backend/src/main/java/io/metersphere/runner/EngineFactory.java new file mode 100644 index 0000000000..7b5b3a2bf7 --- /dev/null +++ b/backend/src/main/java/io/metersphere/runner/EngineFactory.java @@ -0,0 +1,28 @@ +package io.metersphere.runner; + +import io.metersphere.base.domain.FileContent; +import io.metersphere.base.domain.LoadTestWithBLOBs; +import io.metersphere.commons.constants.LoadTestFileType; +import io.metersphere.runner.jmx.JmxEngine; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +public class EngineFactory { + public static Engine createEngine(String engineType) { + final LoadTestFileType type = LoadTestFileType.valueOf(engineType); + + if (type == LoadTestFileType.JMX) { + return new JmxEngine(); + } + return null; + } + + public static EngineContext createContext(LoadTestWithBLOBs loadTest, FileContent fileContent) { + final EngineContext engineContext = new EngineContext(); + engineContext.setEngineId(loadTest.getId()); + engineContext.setInputStream(new ByteArrayInputStream(fileContent.getFile().getBytes(StandardCharsets.UTF_8))); + + return engineContext; + } +} diff --git a/backend/src/main/java/io/metersphere/runner/EngineThread.java b/backend/src/main/java/io/metersphere/runner/EngineThread.java new file mode 100644 index 0000000000..2b3843cae1 --- /dev/null +++ b/backend/src/main/java/io/metersphere/runner/EngineThread.java @@ -0,0 +1,51 @@ +package io.metersphere.runner; + +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class EngineThread implements Runnable { + private Thread thread; + protected volatile boolean stopped = false; + protected boolean isDaemon = false; + + private final AtomicBoolean started = new AtomicBoolean(false); + + public abstract String getEngineName(); + + public void start() { + if (!started.compareAndSet(false, true)) { + return; + } + stopped = false; + this.thread = new Thread(this, getEngineName()); + this.thread.setDaemon(isDaemon); + this.thread.start(); + } + + public void stop() { + this.stop(false); + } + + public void stop(final boolean interrupt) { + if (!started.get()) { + return; + } + this.stopped = true; + + if (interrupt) { + this.thread.interrupt(); + } + } + + + public boolean isStopped() { + return stopped; + } + + public boolean isDaemon() { + return isDaemon; + } + + public void setDaemon(boolean daemon) { + isDaemon = daemon; + } +} \ No newline at end of file diff --git a/backend/src/main/java/io/metersphere/runner/jmx/JmxEngine.java b/backend/src/main/java/io/metersphere/runner/jmx/JmxEngine.java new file mode 100644 index 0000000000..78fc4e36ad --- /dev/null +++ b/backend/src/main/java/io/metersphere/runner/jmx/JmxEngine.java @@ -0,0 +1,106 @@ +package io.metersphere.runner.jmx; + +import io.metersphere.runner.Engine; +import io.metersphere.runner.EngineContext; +import io.metersphere.runner.EngineThread; +import io.metersphere.runner.jmx.client.DistributedRunner; +import io.metersphere.runner.jmx.client.JmeterProperties; +import org.apache.jmeter.JMeter; +import org.apache.jmeter.save.SaveService; +import org.apache.jmeter.services.FileServer; +import org.apache.jmeter.threads.ThreadGroup; +import org.apache.jorphan.collections.HashTree; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.util.Set; + +public class JmxEngine extends EngineThread implements Engine { + private static final Logger log = LoggerFactory.getLogger(JmxEngine.class); + /// todo：从测试属性中读取 + private final static Integer MAX_DURATION = 60; + /// todo：从测试属性中读取 + private final static String REMOTE_HOSTS = "127.0.0.1"; + /// todo：jmeter home如何确定 + private final static String jmeterHome = "/opt/fit2cloud/apache-jmeter-5.2.1"; + private static Method readTreeMethod; + + static { + try { + readTreeMethod = SaveService.class.getDeclaredMethod("readTree", InputStream.class, File.class); + readTreeMethod.setAccessible(true); + } catch (NoSuchMethodException e) { + // ignore + } + } + + private EngineContext context; + private DistributedRunner runner; + + private static void setMaxTestDuration(HashTree jmxTree) { + for (HashTree item : jmxTree.values()) { + Set treeKeys = item.keySet(); + for (Object key : treeKeys) { + if (key instanceof ThreadGroup) { + ((ThreadGroup) key).setProperty(ThreadGroup.SCHEDULER, true); + ((ThreadGroup) key).setProperty(ThreadGroup.DURATION, MAX_DURATION); + } + } + } + } + + @Override + public String getEngineName() { + return "JMX"; + } + + @Override + public boolean init(EngineContext context) { + this.context = context; + + new JmeterProperties(JmxEngine.jmeterHome).initJmeterProperties(); + FileServer.getFileServer().setBaseForScript(new File(JmxEngine.jmeterHome + File.separator + "nothing")); + + final HashTree jmxTree = loadTree(this.context.getInputStream()); + if (jmxTree == null) { + return false; + } + + JMeter.convertSubTree(jmxTree, true); + + setMaxTestDuration(jmxTree); + + this.runner = new DistributedRunner(jmxTree, REMOTE_HOSTS); + + return true; + } + + @Override + public void run() { + try { + this.runner.run(); + } catch (Throwable e) { + log.error("run test error, id: " + this.context.getEngineId(), e); + } + } + + @Override + public void stop() { + super.stop(false); + this.runner.stop(); + } + + private HashTree loadTree(InputStream inputStream) { + try (BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream)) { + return (HashTree) readTreeMethod.invoke(null, bufferedInputStream, null); + } catch (Exception e) { + log.error("Failed to load tree", e); + } + + return null; + } +} diff --git a/backend/src/main/java/io/metersphere/runner/jmx/client/DistributedRunner.java b/backend/src/main/java/io/metersphere/runner/jmx/client/DistributedRunner.java new file mode 100644 index 0000000000..c52fc5508f --- /dev/null +++ b/backend/src/main/java/io/metersphere/runner/jmx/client/DistributedRunner.java @@ -0,0 +1,167 @@ +package io.metersphere.runner.jmx.client; + +import org.apache.jmeter.engine.ClientJMeterEngine; +import org.apache.jmeter.engine.JMeterEngine; +import org.apache.jmeter.report.dashboard.ReportGenerator; +import org.apache.jmeter.samplers.Remoteable; +import org.apache.jmeter.testelement.TestStateListener; +import org.apache.jmeter.util.JMeterUtils; +import org.apache.jorphan.collections.HashTree; +import org.apache.jorphan.util.JOrphanUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class DistributedRunner extends org.apache.jmeter.engine.DistributedRunner { + private static final String HOSTS_SEPARATOR = ","; + private HashTree jmxTree; + private String hosts; + + public DistributedRunner(HashTree jmxTree, String hosts) { + this.jmxTree = jmxTree; + this.hosts = hosts; + } + + public void run() { + final List hosts = getRemoteHosts(); + final ListenToTest listener = new ListenToTest(false, null); + jmxTree.add(jmxTree.getArray()[0], listener); + init(hosts, jmxTree); + listener.setStartedRemoteEngines(new ArrayList<>(getEngines())); + start(); + } + + private List getRemoteHosts() { + StringTokenizer st = new StringTokenizer(hosts, HOSTS_SEPARATOR); + List list = new LinkedList<>(); + while (st.hasMoreElements()) { + list.add((String) st.nextElement()); + } + return list; + } + + private static class ListenToTest implements TestStateListener, Remoteable { + private final Logger log = LoggerFactory.getLogger(ListenToTest.class); + private final ReportGenerator reportGenerator; + private AtomicInteger startedRemoteEngines = new AtomicInteger(0); + private ConcurrentLinkedQueue remoteEngines = new ConcurrentLinkedQueue<>(); + private boolean remoteStop; + + ListenToTest(boolean remoteStop, ReportGenerator reportGenerator) { + this.remoteStop = remoteStop; + this.reportGenerator = reportGenerator; + } + + void setStartedRemoteEngines(List engines) { + this.remoteEngines.clear(); + this.remoteEngines.addAll(engines); + this.startedRemoteEngines = new AtomicInteger(remoteEngines.size()); + } + + @Override + // N.B. this is called by a daemon RMI thread from the remote host + public void testEnded(String host) { + final long now = System.currentTimeMillis(); + log.info("Finished remote host: {} ({})", host, now); + if (startedRemoteEngines.decrementAndGet() <= 0) { + log.info("All remote engines have ended test, starting RemoteTestStopper thread"); + Thread stopSoon = new Thread(() -> endTest(true), "RemoteTestStopper"); + // the calling thread is a daemon; this thread must not be + // see Bug 59391 + stopSoon.setDaemon(false); + stopSoon.start(); + } + } + + @Override + public void testEnded() { + endTest(false); + } + + @Override + public void testStarted(String host) { + final long now = System.currentTimeMillis(); + log.info("Started remote host: {} ({})", host, now); + } + + @Override + public void testStarted() { + if (log.isInfoEnabled()) { + final long now = System.currentTimeMillis(); + log.info("{} ({})", JMeterUtils.getResString("running_test"), now);//$NON-NLS-1$ + } + } + + private void endTest(boolean isDistributed) { + long now = System.currentTimeMillis(); + if (isDistributed) { + log.info("Tidying up remote @ " + new Date(now) + " (" + now + ")"); + } else { + log.info("Tidying up ... @ " + new Date(now) + " (" + now + ")"); + } + + if (isDistributed) { + if (remoteStop) { + log.info("Exiting remote servers:" + remoteEngines); + for (JMeterEngine engine : remoteEngines) { + log.info("Exiting remote server:" + engine); + engine.exit(); + } + } + try { + TimeUnit.SECONDS.sleep(5); // Allow listeners to close files + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + ClientJMeterEngine.tidyRMI(log); + } + + if (reportGenerator != null) { + try { + log.info("Generating Dashboard"); + reportGenerator.generate(); + log.info("Dashboard generated"); + } catch (Exception ex) { + System.err.println("Error generating the report: " + ex);//NOSONAR + log.error("Error generating the report: {}", ex.getMessage(), ex); + } + } + checkForRemainingThreads(); + log.info("... end of run"); + } + + /** + * Runs daemon thread which waits a short while; + * if JVM does not exit, lists remaining non-daemon threads on stdout. + */ + private void checkForRemainingThreads() { + // This cannot be a JMeter class variable, because properties + // are not initialised until later. + final int pauseToCheckForRemainingThreads = + JMeterUtils.getPropDefault("jmeter.exit.check.pause", 2000); // $NON-NLS-1$ + + if (pauseToCheckForRemainingThreads > 0) { + Thread daemon = new Thread(() -> { + try { + TimeUnit.MILLISECONDS.sleep(pauseToCheckForRemainingThreads); // Allow enough time for JVM to exit + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + // This is a daemon thread, which should only reach here if there are other + // non-daemon threads still active + System.out.println("The JVM should have exited but did not.");//NOSONAR + System.out.println("The following non-daemon threads are still running (DestroyJavaVM is OK):");//NOSONAR + JOrphanUtils.displayThreads(false); + }); + daemon.setDaemon(true); + daemon.start(); + } else if (pauseToCheckForRemainingThreads <= 0) { + log.debug("jmeter.exit.check.pause is <= 0, JMeter won't check for unterminated non-daemon threads"); + } + } + } +} \ No newline at end of file diff --git a/backend/src/main/java/io/metersphere/runner/jmx/client/JmeterProperties.java b/backend/src/main/java/io/metersphere/runner/jmx/client/JmeterProperties.java new file mode 100644 index 0000000000..b2be78e7bf --- /dev/null +++ b/backend/src/main/java/io/metersphere/runner/jmx/client/JmeterProperties.java @@ -0,0 +1,70 @@ +package io.metersphere.runner.jmx.client; + +import org.apache.jmeter.util.JMeterUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; + +public class JmeterProperties extends Properties { + private final static Logger logger = LoggerFactory.getLogger(JmeterProperties.class); + + private final String jmeterHome; + + public JmeterProperties(String jmeterHome) { + this.jmeterHome = jmeterHome; + } + + public void initJmeterProperties() { + JMeterUtils.loadJMeterProperties(getJmeterHomeBin() + File.separator + "jmeter.properties"); + JMeterUtils.setJMeterHome(getJmeterHome()); + JMeterUtils.initLocale(); + + Properties jmeterProps = JMeterUtils.getJMeterProperties(); + + // Add local JMeter properties, if the file is found + String userProp = JMeterUtils.getPropDefault("user.properties", ""); + if (userProp.length() > 0) { + File file = JMeterUtils.findFile(userProp); + if (file.canRead()) { + try (FileInputStream fis = new FileInputStream(file)) { + Properties tmp = new Properties(); + tmp.load(fis); + jmeterProps.putAll(tmp); + } catch (IOException e) { + logger.error("Failed to init jmeter properties", e); + } + } + } + + // Add local system properties, if the file is found + String sysProp = JMeterUtils.getPropDefault("system.properties", ""); + if (sysProp.length() > 0) { + File file = JMeterUtils.findFile(sysProp); + if (file.canRead()) { + try (FileInputStream fis = new FileInputStream(file)) { + System.getProperties().load(fis); + } catch (IOException e) { + logger.error("Failed to init jmeter properties", e); + } + } + } + + jmeterProps.put("jmeter.version", JMeterUtils.getJMeterVersion()); + for (Map.Entry entry : jmeterProps.entrySet()) { + put(entry.getKey(), entry.getValue()); + } + } + + private String getJmeterHome() { + return jmeterHome; + } + + private String getJmeterHomeBin() { + return getJmeterHome() + File.separator + "bin"; + } +} \ No newline at end of file diff --git a/backend/src/main/java/io/metersphere/service/FileService.java b/backend/src/main/java/io/metersphere/service/FileService.java index c10489c013..d30a2eab51 100644 --- a/backend/src/main/java/io/metersphere/service/FileService.java +++ b/backend/src/main/java/io/metersphere/service/FileService.java @@ -64,6 +64,10 @@ public class FileService { return fileMetadataMapper.selectByPrimaryKey(loadTestFiles.get(0).getFileId()); } + public FileContent getFileContent(String fileId) { + return fileContentMapper.selectByPrimaryKey(fileId); + } + public void deleteFileByTestId(String testId) { LoadTestFileExample loadTestFileExample = new LoadTestFileExample(); loadTestFileExample.createCriteria().andTestIdEqualTo(testId); diff --git a/backend/src/main/java/io/metersphere/service/LoadTestService.java b/backend/src/main/java/io/metersphere/service/LoadTestService.java index 11ba8322a0..56c0af6ac0 100644 --- a/backend/src/main/java/io/metersphere/service/LoadTestService.java +++ b/backend/src/main/java/io/metersphere/service/LoadTestService.java @@ -8,6 +8,8 @@ import io.metersphere.commons.exception.MSException; import io.metersphere.commons.utils.IOUtils; import io.metersphere.controller.request.testplan.*; import io.metersphere.dto.LoadTestDTO; +import io.metersphere.runner.Engine; +import io.metersphere.runner.EngineFactory; import org.apache.commons.lang3.RandomUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -151,6 +153,31 @@ public class LoadTestService { MSException.throwException("无法运行测试，未找到测试：" + request.getId()); } + final FileMetadata fileMetadata = fileService.getFileMetadataByTestId(request.getId()); + if (fileMetadata == null) { + MSException.throwException("无法运行测试，无法获取测试文件元信息，测试ID：" + request.getId()); + } + + final FileContent fileContent = fileService.getFileContent(fileMetadata.getId()); + if (fileContent == null) { + MSException.throwException("无法运行测试，无法获取测试文件内容，测试ID：" + request.getId()); + } + System.out.println("开始运行：" + loadTest.getName()); + final Engine engine = EngineFactory.createEngine(fileMetadata.getType()); + if (engine == null) { + MSException.throwException(String.format("无法运行测试，未识别测试文件类型，测试ID：%s，文件类型：%s", + request.getId(), + fileMetadata.getType())); + } + + final boolean init = engine.init(EngineFactory.createContext(loadTest, fileContent)); + if (!init) { + MSException.throwException(String.format("无法运行测试，初始化运行环境失败，测试ID：%s", request.getId())); + } + + engine.start(); + + /// todo：通过调用stop方法能够停止正在运行的engine，但是如果部署了多个backend实例，页面发送的停止请求如何定位到具体的engine } }