opensci
/
piflow
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

optimize code

This commit is contained in:
judy0131 2020-03-23 19:17:12 +08:00
parent b3e1f9b0d1
commit 1fd1dae8b1
1 changed files with 1 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
piflow-core/src/main/scala/cn/piflow/util/FlowLauncher.scala
View File

@ -29,8 +29,6 @@ object FlowLauncher {
.setDeployMode(PropertyUtil.getPropertyValue("spark.deploy.mode")) .setDeployMode(PropertyUtil.getPropertyValue("spark.deploy.mode"))
.setAppResource(ConfigureUtil.getPiFlowBundlePath()) .setAppResource(ConfigureUtil.getPiFlowBundlePath())
.setVerbose(true) .setVerbose(true)
//.setConf("spark.jars", PropertyUtil.getPropertyValue("piflow.bundle"))
//.setConf("spark.hive.metastore.uris",PropertyUtil.getPropertyValue("hive.metastore.uris"))
.setConf("spark.driver.memory", flow.getDriverMemory()) .setConf("spark.driver.memory", flow.getDriverMemory())
.setConf("spark.executor.instances", flow.getExecutorNum()) .setConf("spark.executor.instances", flow.getExecutorNum())
.setConf("spark.executor.memory", flow.getExecutorMem()) .setConf("spark.executor.memory", flow.getExecutorMem())
@ -40,24 +38,9 @@ object FlowLauncher {
.setMainClass("cn.piflow.api.StartFlowMain") .setMainClass("cn.piflow.api.StartFlowMain")
.addAppArgs(flowJson) .addAppArgs(flowJson)
/*if (PropertyUtil.getPropertyValue("hive.metastore.uris") != null){
sparkLauncher.setConf("spark.hive.metastore.uris",PropertyUtil.getPropertyValue("hive.metastore.uris"))
}*/
if(PropertyUtil.getPropertyValue("yarn.resourcemanager.hostname") != null) if(PropertyUtil.getPropertyValue("yarn.resourcemanager.hostname") != null)
sparkLauncher.setConf("spark.hadoop.yarn.resourcemanager.hostname", PropertyUtil.getPropertyValue("yarn.resourcemanager.hostname")) sparkLauncher.setConf("spark.hadoop.yarn.resourcemanager.hostname", PropertyUtil.getPropertyValue("yarn.resourcemanager.hostname"))
/*if(PropertyUtil.getPropertyValue("yarn.resourcemanager.address") != null)
sparkLauncher.setConf("spark.hadoop.yarn.resourcemanager.address", PropertyUtil.getPropertyValue("yarn.resourcemanager.address"))
if(PropertyUtil.getPropertyValue("spark.yarn.access.namenode") != null)
sparkLauncher.setConf("spark.yarn.access.namenode", PropertyUtil.getPropertyValue("spark.yarn.access.namenode"))
else
sparkLauncher.setConf("spark.yarn.access.namenode", PropertyUtil.getPropertyValue("fs.defaultFS"))
if(PropertyUtil.getPrope rtyValue("yarn.stagingDir") != null)
sparkLauncher.setConf("spark.yarn.stagingDir", PropertyUtil.getPropertyValue("yarn.stagingDir"))
if(PropertyUtil.getPropertyValue("yarn.jars") != null)
sparkLauncher.setConf("spark.yarn.jars", PropertyUtil.getPropertyValue("yarn.jars"))*/
//add other jars for application //add other jars for application
val classPath = PropertyUtil.getClassPath() val classPath = PropertyUtil.getClassPath()