yystopf
/
piflow
forked from opensci/piflow
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add yarn.jars config to solve the problem -->"Neither spark.yarn.jars nor spark.yarn.archive is set, falling back to uploading libraries under SPARK_HOME",which makes the spark on yarn job run very slow!

This commit is contained in:
judy0131 2018-09-13 10:16:22 +08:00
parent 6b72da8518
commit 11a594d20c
2 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
conf/config.properties
View File

@ -9,6 +9,7 @@ yarn.resourcemanager.hostname=10.0.86.191
yarn.resourcemanager.address=10.0.86.191:8032
yarn.access.namenode=hdfs://10.0.86.191:9000
yarn.stagingDir=hdfs://10.0.86.191:9000/tmp/
yarn.jars=hdfs://10.0.86.191:9000/user/spark/share/lib/*.jar
hive.metastore.uris=thrift://10.0.86.191:9083

3
piflow-server/src/main/scala/cn/piflow/api/API.scala
View File

@ -32,11 +32,12 @@ object API {
.config("spark.hadoop.yarn.resourcemanager.address", PropertyUtil.getPropertyValue("yarn.resourcemanager.address"))
.config("spark.yarn.access.namenode", PropertyUtil.getPropertyValue("yarn.access.namenode"))
.config("spark.yarn.stagingDir", PropertyUtil.getPropertyValue("yarn.stagingDir"))
.config("spark.yarn.jars", PropertyUtil.getPropertyValue("yarn.jars"))
//.config("spark.driver.memory", "1g")
//.config("spark.executor.memory", "1g")
//.config("spark.cores.max", "2")
.config("spark.jars", PropertyUtil.getPropertyValue("piflow.bundle"))
.config("hive.metastore.uris",PropertyUtil.getPropertyValue("hive.metastore.uris"))
//.config("hive.metastore.uris",PropertyUtil.getPropertyValue("hive.metastore.uris"))
.enableHiveSupport()
.getOrCreate()