启动测试 0.1

This commit is contained in:
Captain.B 2020-03-16 17:13:42 +08:00
parent 454320f9a4
commit 4eaaa746df
15 changed files with 583 additions and 7 deletions

View File

@ -4,18 +4,27 @@ import java.util.HashMap;
import java.util.Map;
public class EngineContext {
private String engineId;
private String testId;
private String namespace;
private String engineType;
private String fileType;
private String content;
private Map<String, Object> properties = new HashMap<>();
public String getEngineId() {
return engineId;
public String getTestId() {
return testId;
}
public void setEngineId(String engineId) {
this.engineId = engineId;
public void setTestId(String testId) {
this.testId = testId;
}
public String getNamespace() {
return namespace;
}
public void setNamespace(String namespace) {
this.namespace = namespace;
}
public String getEngineType() {

View File

@ -30,7 +30,8 @@ public class EngineFactory {
public static EngineContext createContext(LoadTestWithBLOBs loadTest, FileMetadata fileMetadata, FileContent fileContent) throws Exception {
final EngineContext engineContext = new EngineContext();
engineContext.setEngineId(loadTest.getId());
engineContext.setTestId(loadTest.getId());
engineContext.setNamespace(loadTest.getProjectId());
engineContext.setEngineType(fileMetadata.getEngine());
engineContext.setFileType(fileMetadata.getType());

View File

@ -1,7 +1,16 @@
package io.metersphere.engine.kubernetes;
import com.alibaba.fastjson.JSON;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.metersphere.commons.utils.LogUtil;
import io.metersphere.engine.Engine;
import io.metersphere.engine.EngineContext;
import io.metersphere.engine.kubernetes.crds.jmeter.Jmeter;
import io.metersphere.engine.kubernetes.provider.ClientCredential;
import io.metersphere.engine.kubernetes.provider.KubernetesProvider;
import java.util.HashMap;
public class KubernetesTestEngine implements Engine {
private EngineContext context;
@ -16,7 +25,33 @@ public class KubernetesTestEngine implements Engine {
@Override
public void start() {
// todo 运行测试
ClientCredential credential = new ClientCredential();
credential.setMasterUrl("https://172.16.10.93:6443");
KubernetesProvider kubernetesProvider = new KubernetesProvider(JSON.toJSONString(credential));
// create cm
try (KubernetesClient client = kubernetesProvider.getKubernetesClient()) {
String configMapName = context.getTestId() + "-files";
ConfigMap configMap = client.configMaps().inNamespace(context.getNamespace()).withName(configMapName).get();
if (configMap == null) {
ConfigMap item = new ConfigMap();
item.setData(new HashMap<String, String>() {{
put("sample.jmx", context.getContent());
}});
client.configMaps().inNamespace(context.getNamespace()).create(item);
}
}
// create jmeter
try {
Jmeter jmeter = new Jmeter();
jmeter.getMetadata().setNamespace(context.getNamespace());
jmeter.getMetadata().setName(context.getTestId());
jmeter.getSpec().setReplicas(1);
jmeter.getSpec().setImage("registry.fit2cloud.com/metersphere/jmeter:0.0.2");
kubernetesProvider.applyCustomResource(jmeter);
} catch (Exception e) {
LogUtil.error(e);
}
}
@Override

View File

@ -0,0 +1,36 @@
package io.metersphere.engine.kubernetes.crds;
import io.fabric8.kubernetes.client.CustomResource;
public class MeterSphereCustomResource extends CustomResource {
private String crd;
private Object spec;
private Object status;
public String getCrd() {
return crd;
}
public void setCrd(String crd) {
this.crd = crd;
}
public Object getSpec() {
return spec;
}
public void setSpec(Object spec) {
this.spec = spec;
}
public Object getStatus() {
return status;
}
public void setStatus(Object status) {
this.status = status;
}
}

View File

@ -0,0 +1,10 @@
package io.metersphere.engine.kubernetes.crds;
import io.fabric8.kubernetes.api.builder.Function;
import io.fabric8.kubernetes.client.CustomResourceDoneable;
public class MeterSphereCustomResourceDoneable<T extends MeterSphereCustomResource> extends CustomResourceDoneable<T> {
public MeterSphereCustomResourceDoneable(T resource, Function<T, T> function) {
super(resource, function);
}
}

View File

@ -0,0 +1,6 @@
package io.metersphere.engine.kubernetes.crds;
import io.fabric8.kubernetes.client.CustomResourceList;
public class MeterSphereCustomResourceList<T extends MeterSphereCustomResource> extends CustomResourceList<T> {
}

View File

@ -0,0 +1,33 @@
package io.metersphere.engine.kubernetes.crds.jmeter;
import io.metersphere.engine.kubernetes.crds.MeterSphereCustomResource;
public class Jmeter extends MeterSphereCustomResource {
public static final String CRD = "jmeters.metersphere.io";
public static final String KIND = "Jmeter";
private JmeterSpec spec;
private JmeterStatus status;
public Jmeter() {
this.setCrd(CRD);
this.setKind(KIND);
}
@Override
public JmeterSpec getSpec() {
return spec;
}
public void setSpec(JmeterSpec spec) {
this.spec = spec;
}
@Override
public JmeterStatus getStatus() {
return status;
}
public void setStatus(JmeterStatus status) {
this.status = status;
}
}

View File

@ -0,0 +1,10 @@
package io.metersphere.engine.kubernetes.crds.jmeter;
import io.fabric8.kubernetes.api.builder.Function;
import io.fabric8.kubernetes.client.CustomResourceDoneable;
public class JmeterDoneable extends CustomResourceDoneable<Jmeter> {
public JmeterDoneable(Jmeter resource, Function<Jmeter, Jmeter> function) {
super(resource, function);
}
}

View File

@ -0,0 +1,28 @@
package io.metersphere.engine.kubernetes.crds.jmeter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import io.fabric8.kubernetes.api.model.KubernetesResource;
@JsonDeserialize
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public class JmeterSpec implements KubernetesResource {
private int replicas = 1;
private String image;
public int getReplicas() {
return replicas;
}
public void setReplicas(int replicas) {
this.replicas = replicas;
}
public String getImage() {
return image;
}
public void setImage(String image) {
this.image = image;
}
}

View File

@ -0,0 +1,36 @@
package io.metersphere.engine.kubernetes.crds.jmeter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import io.fabric8.kubernetes.api.model.KubernetesResource;
@JsonDeserialize
@JsonInclude(JsonInclude.Include.NON_NULL)
public class JmeterStatus implements KubernetesResource {
private String phase;
private String reason;
public String getPhase() {
return phase;
}
public void setPhase(String phase) {
this.phase = phase;
}
public String getReason() {
return reason;
}
public void setReason(String reason) {
this.reason = reason;
}
@Override
public String toString() {
return "JmeterStatus{" +
"phase='" + phase + '\'' +
", reason='" + reason + '\'' +
'}';
}
}

View File

@ -0,0 +1,268 @@
package io.metersphere.engine.kubernetes.provider;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.fabric8.kubernetes.api.model.*;
import io.fabric8.kubernetes.api.model.apiextensions.CustomResourceDefinition;
import io.fabric8.kubernetes.client.ConfigBuilder;
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.dsl.*;
import io.metersphere.engine.kubernetes.crds.MeterSphereCustomResource;
import io.metersphere.engine.kubernetes.crds.MeterSphereCustomResourceDoneable;
import io.metersphere.engine.kubernetes.crds.MeterSphereCustomResourceList;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.*;
import java.util.regex.Pattern;
public abstract class AbstractClientProvider {
private static ObjectMapper objectMapper = new ObjectMapper();
private String credential;
public AbstractClientProvider(String credential) {
setCredential(credential);
}
public String getCredential() {
return credential;
}
public void setCredential(String credential) {
this.credential = credential;
}
/**
* OpenShiftClient继承自KubernetesClientOpenShiftClient对OpenShift和kubernetes的共有资源也是用KubernetesClient代理的
* 所以可以在把client提取到抽象类
*/
public KubernetesClient getKubernetesClient() {
ClientCredential providerCredential = JSONObject.parseObject(getCredential(), ClientCredential.class);
io.fabric8.kubernetes.client.ConfigBuilder configBuilder = new ConfigBuilder();
configBuilder.withMasterUrl(providerCredential.getMasterUrl());
configBuilder.withOauthToken(providerCredential.getToken());
configBuilder.withTrustCerts(true);
//设置默认的 namespace null
configBuilder.withNamespace(null);
return new DefaultKubernetesClient(configBuilder.build());
}
public void validateCredential() {
try (KubernetesClient client = getKubernetesClient()) {
client.namespaces().list();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* 确保指定的namespace存在不存在直接创建一个
*
* @param namespace namespace标识
* @return
*/
public synchronized String confirmNamespace(String namespace, String displayName) {
KubernetesClient kubernetesClient = getKubernetesClient();
Namespace currentNamespace = kubernetesClient.namespaces().withName(namespace).get();
if (currentNamespace == null) {
Map<String, String> annotations = new HashMap<>();
annotations.put("fit2cloud.com/workspace-name", displayName);
Namespace newNamespace = new NamespaceBuilder()
.withNewMetadata()
.withName(namespace)
.withAnnotations(annotations)
.endMetadata()
.build();
currentNamespace = kubernetesClient.namespaces().createOrReplace(newNamespace);
}
return currentNamespace.getMetadata().getName();
}
/**
* 获取集群UUID当前集群UUID等于default namespace的UID
*
* @return
*/
public String getClusterUUID() {
KubernetesClient kubernetesClient = getKubernetesClient();
Namespace defaultNamespace = kubernetesClient.namespaces().withName("kube-system").get();
if (defaultNamespace == null) {
throw new RuntimeException("无法获取集群的kube-system namespace");
} else {
return defaultNamespace.getMetadata().getUid();
}
}
/**
* 确保docker registry存在
* 不存在 创建一个harbor-secret并修改serviceaccount default
*
* @param registry
*/
public void dockerRegistry(DockerRegistry registry) {
if (StringUtils.isEmpty(registry.getUsername()) ||
StringUtils.isEmpty(registry.getPassword()) ||
StringUtils.isEmpty(registry.getUrl())
) {
throw new RuntimeException("Please set the docker registry information");
}
String secretName = "docker-registry-ms-secret";
KubernetesClient kubernetesClient = getKubernetesClient();
Secret secretRegistry = new SecretBuilder()
.withNewMetadata().withName(secretName).endMetadata()
.addToData(".dockerconfigjson", DockerRegistryUtil.getDockerConfig(registry))
.withType("kubernetes.io/dockerconfigjson")
.build();
kubernetesClient.secrets().inNamespace(registry.getNamespace()).createOrReplace(secretRegistry);
//sa
ServiceAccount serviceAccount = kubernetesClient.serviceAccounts().inNamespace(registry.getNamespace())
.withName("default").get();
List<LocalObjectReference> imagePullSecrets = serviceAccount.getImagePullSecrets();
for (LocalObjectReference pullSecret : imagePullSecrets) {
if (secretName.equals(pullSecret.getName())) {
return;
}
}
LocalObjectReference localObjectReference = new LocalObjectReference(secretName);
imagePullSecrets.add(localObjectReference);
serviceAccount.setImagePullSecrets(imagePullSecrets);
kubernetesClient.serviceAccounts().inNamespace(registry.getNamespace())
.createOrReplace(serviceAccount);
}
public <T> T applyCustomResource(MeterSphereCustomResource customResource) throws Exception {
try (KubernetesClient kubernetesClient = getKubernetesClient()) {
CustomResourceDefinition crd = kubernetesClient.customResourceDefinitions().withName(customResource.getCrd()).get();
if (crd == null) {
throw new Exception("CRD does not exists.");
}
MixedOperation<MeterSphereCustomResource, MeterSphereCustomResourceList, MeterSphereCustomResourceDoneable, Resource<MeterSphereCustomResource, MeterSphereCustomResourceDoneable>>
operation = kubernetesClient.customResources(crd, MeterSphereCustomResource.class, MeterSphereCustomResourceList.class, MeterSphereCustomResourceDoneable.class);
MeterSphereCustomResource replace = operation.inNamespace(customResource.getMetadata().getNamespace()).createOrReplace(customResource);
return (T) objectMapper.readValue(objectMapper.writeValueAsString(replace), customResource.getClass());
}
}
public boolean deleteCustomResource(MeterSphereCustomResource customResource) {
try (KubernetesClient kubernetesClient = getKubernetesClient()) {
CustomResourceDefinition crd = kubernetesClient.customResourceDefinitions().withName(customResource.getCrd()).get();
MixedOperation<MeterSphereCustomResource, MeterSphereCustomResourceList, MeterSphereCustomResourceDoneable, Resource<MeterSphereCustomResource, MeterSphereCustomResourceDoneable>>
operation = kubernetesClient.customResources(crd, MeterSphereCustomResource.class, MeterSphereCustomResourceList.class, MeterSphereCustomResourceDoneable.class);
Boolean result = operation.inNamespace(customResource.getMetadata().getNamespace()).withName(customResource.getMetadata().getName()).cascading(true).delete();
return result == null ? false : result;
}
}
public <T> T getCustomResource(MeterSphereCustomResource customResource) throws Exception {
try (KubernetesClient kubernetesClient = getKubernetesClient()) {
CustomResourceDefinition crd = kubernetesClient.customResourceDefinitions().withName(customResource.getCrd()).get();
MixedOperation<MeterSphereCustomResource, MeterSphereCustomResourceList, MeterSphereCustomResourceDoneable, Resource<MeterSphereCustomResource, MeterSphereCustomResourceDoneable>>
operation = kubernetesClient.customResources(crd, MeterSphereCustomResource.class, MeterSphereCustomResourceList.class, MeterSphereCustomResourceDoneable.class);
MeterSphereCustomResource meterSphereCustomResource = operation.inNamespace(customResource.getMetadata().getNamespace()).withName(customResource.getMetadata().getName()).get();
if (meterSphereCustomResource == null) {
return null;
}
return (T) objectMapper.readValue(objectMapper.writeValueAsString(meterSphereCustomResource), customResource.getClass());
}
}
public <T> List<T> listCustomResource(MeterSphereCustomResource customResource) throws Exception {
try (KubernetesClient kubernetesClient = getKubernetesClient()) {
CustomResourceDefinition crd = kubernetesClient.customResourceDefinitions().withName(customResource.getCrd()).get();
MixedOperation<MeterSphereCustomResource, MeterSphereCustomResourceList, MeterSphereCustomResourceDoneable, Resource<MeterSphereCustomResource, MeterSphereCustomResourceDoneable>>
operation = kubernetesClient.customResources(crd, MeterSphereCustomResource.class, MeterSphereCustomResourceList.class, MeterSphereCustomResourceDoneable.class);
MeterSphereCustomResourceList list;
if (StringUtils.isNotEmpty(customResource.getMetadata().getNamespace())) {
list = operation.inNamespace(customResource.getMetadata().getNamespace()).list();
} else {
list = operation.inAnyNamespace().list();
}
List<T> resultList = new ArrayList<>();
for (Object cr : list.getItems()) {
resultList.add((T) objectMapper.readValue(objectMapper.writeValueAsString(cr), customResource.getClass()));
}
return resultList;
}
}
public boolean checkPVCNotExists(String namespace, String statefulsetName) {
KubernetesClient kubernetesClient = getKubernetesClient();
NonNamespaceOperation<PersistentVolumeClaim, PersistentVolumeClaimList, DoneablePersistentVolumeClaim, Resource<PersistentVolumeClaim, DoneablePersistentVolumeClaim>> operation = kubernetesClient.persistentVolumeClaims()
.inNamespace(namespace);
PersistentVolumeClaimList pvcList = operation.list(100, null);
Pattern compile = Pattern.compile(statefulsetName + "-\\d+");
return checkPVCNotExists(pvcList, compile, operation);
}
private boolean checkPVCNotExists(PersistentVolumeClaimList pvcList, Pattern compile, NonNamespaceOperation<PersistentVolumeClaim, PersistentVolumeClaimList, DoneablePersistentVolumeClaim, Resource<PersistentVolumeClaim, DoneablePersistentVolumeClaim>> operation) {
if (pvcList == null || CollectionUtils.isEmpty(pvcList.getItems())) {
return true;
}
Optional<PersistentVolumeClaim> claimOptional = pvcList.getItems().stream().filter(pvc -> compile.matcher(pvc.getMetadata().getName()).matches()).findAny();
if (claimOptional.isPresent()) {
return false;
} else if (StringUtils.isNotEmpty(pvcList.getMetadata().getContinue())) {
return checkPVCNotExists(operation.list(100, pvcList.getMetadata().getContinue()), compile, operation);
} else {
return true;
}
}
public boolean deletePVC(HasMetadata hasMetadata) {
KubernetesClient kubernetesClient = getKubernetesClient();
NonNamespaceOperation<PersistentVolumeClaim, PersistentVolumeClaimList, DoneablePersistentVolumeClaim, Resource<PersistentVolumeClaim, DoneablePersistentVolumeClaim>> operation = kubernetesClient.persistentVolumeClaims()
.inNamespace(hasMetadata.getMetadata().getNamespace());
if (MapUtils.isNotEmpty(hasMetadata.getMetadata().getLabels())) {
operation.withLabelSelector(new LabelSelector(null, hasMetadata.getMetadata().getLabels()));
}
if (StringUtils.isNotEmpty(hasMetadata.getMetadata().getName())) {
operation.withName(hasMetadata.getMetadata().getName());
}
Boolean delete = operation.delete();
return delete == null ? false : delete;
}
public boolean deleteNamespace(String namespace) {
KubernetesClient kubernetesClient = getKubernetesClient();
Boolean delete = kubernetesClient.namespaces().withName(namespace).delete();
return delete == null ? false : delete;
}
public String getLog(String namespace, String pod, String container, int tailingLines) {
try (KubernetesClient client = getKubernetesClient()) {
PrettyLoggable<String, LogWatch> loggable;
if (tailingLines > 0) {
loggable = client.pods().inNamespace(namespace).withName(pod).inContainer(container).tailingLines(tailingLines);
} else {
loggable = client.pods().inNamespace(namespace).withName(pod).inContainer(container);
}
return loggable.getLog();
}
}
}

View File

@ -0,0 +1,23 @@
package io.metersphere.engine.kubernetes.provider;
public class ClientCredential {
private String masterUrl;
private String token;
public String getMasterUrl() {
return masterUrl;
}
public void setMasterUrl(String masterUrl) {
this.masterUrl = masterUrl;
}
public String getToken() {
return token;
}
public void setToken(String token) {
this.token = token;
}
}

View File

@ -0,0 +1,50 @@
package io.metersphere.engine.kubernetes.provider;
public class DockerRegistry {
private String url;
private String username;
private String password;
private String email;
private String namespace;
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getEmail() {
return email;
}
public void setEmail(String email) {
this.email = email;
}
public String getNamespace() {
return namespace;
}
public void setNamespace(String namespace) {
this.namespace = namespace;
}
}

View File

@ -0,0 +1,24 @@
package io.metersphere.engine.kubernetes.provider;
import com.alibaba.fastjson.JSONObject;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
public class DockerRegistryUtil {
public static String getDockerConfig(DockerRegistry registry) {
Map<String, String> config = new HashMap<>();
config.put("username", registry.getUsername());
config.put("password", registry.getPassword());
config.put("auth", Base64.getEncoder().encodeToString((registry.getUsername() + ":" + registry.getPassword()).getBytes()));
JSONObject jb = new JSONObject();
jb.put(registry.getUrl(), config);
JSONObject result = new JSONObject();
result.put("auths", jb);
return Base64.getEncoder().encodeToString(result.toJSONString().getBytes());
}
}

View File

@ -0,0 +1,7 @@
package io.metersphere.engine.kubernetes.provider;
public class KubernetesProvider extends AbstractClientProvider {
public KubernetesProvider(String credential) {
super(credential);
}
}