forked from opensci/piflow
fix checkpoint bug
This commit is contained in:
parent
a70fe6a989
commit
554cd25dc4
|
@ -188,7 +188,7 @@ class NewXmlParser extends ConfigurableStop{
|
|||
descriptor
|
||||
}
|
||||
override def getIcon(): Array[Byte] = {
|
||||
ImageUtil.getImage("./src/main/resources/ShellExecutor.jpg")
|
||||
ImageUtil.getImage("xml.png")
|
||||
}
|
||||
override def getGroup(): List[String] = {
|
||||
List(StopGroupEnum.XmlGroup.toString)
|
||||
|
|
|
@ -88,6 +88,8 @@ object API {
|
|||
.setConf("spark.driver.memory", "1g")
|
||||
.setConf("spark.executor.memory", "1g")
|
||||
.setConf("spark.cores.max", "2")
|
||||
//.setConf("spark.checkpoint", PropertyUtil.getPropertyValue("checkpoint.path"))
|
||||
.addFile(PropertyUtil.getConfigureFile())
|
||||
.setMainClass("cn.piflow.api.StartFlowMain")
|
||||
.addAppArgs(flowJson.stripMargin)
|
||||
//.redirectOutput(stdout)
|
||||
|
|
|
@ -25,11 +25,11 @@ object StartFlowMain {
|
|||
.appName(flowBean.name)
|
||||
.enableHiveSupport()
|
||||
.getOrCreate()
|
||||
//val checkpointPath = spark.sparkContext.getConf.get("checkpoint.path")
|
||||
|
||||
val process = Runner.create()
|
||||
.bind(classOf[SparkSession].getName, spark)
|
||||
//.bind("checkpoint.path", PropertyUtil.getPropertyValue("checkpoint.path"))
|
||||
.bind("checkpoint.path","hdfs://10.0.86.89:9000/xjzhu/piflow/checkpoints/")
|
||||
.bind("checkpoint.path",PropertyUtil.getPropertyValue("checkpoint.path"))
|
||||
.start(flow);
|
||||
val applicationId = spark.sparkContext.applicationId
|
||||
process.awaitTermination();
|
||||
|
|
|
@ -6,16 +6,21 @@ import java.util.Properties
|
|||
object PropertyUtil {
|
||||
private val prop: Properties = new Properties()
|
||||
var fis: InputStream = null
|
||||
try{
|
||||
var path :String = ""
|
||||
try{
|
||||
//val path = Thread.currentThread().getContextClassLoader.getResource("config.properties").getPath
|
||||
//fis = this.getClass.getResourceAsStream("")
|
||||
val userDir = System.getProperty("user.dir")
|
||||
val path = userDir + "/conf/" + "config.properties"
|
||||
path = userDir + "/conf/" + "config.properties"
|
||||
prop.load(new FileInputStream(path))
|
||||
} catch{
|
||||
case ex: Exception => ex.printStackTrace()
|
||||
}
|
||||
|
||||
def getConfigureFile() : String = {
|
||||
path
|
||||
}
|
||||
|
||||
def getPropertyValue(propertyKey: String): String ={
|
||||
val obj = prop.get(propertyKey)
|
||||
if(obj != null){
|
||||
|
|
Loading…
Reference in New Issue