fix checkpoint bug

This commit is contained in:
judy0131 2018-10-22 12:22:55 +08:00
parent a70fe6a989
commit 554cd25dc4
4 changed files with 12 additions and 5 deletions

View File

@ -188,7 +188,7 @@ class NewXmlParser extends ConfigurableStop{
descriptor
}
override def getIcon(): Array[Byte] = {
ImageUtil.getImage("./src/main/resources/ShellExecutor.jpg")
ImageUtil.getImage("xml.png")
}
override def getGroup(): List[String] = {
List(StopGroupEnum.XmlGroup.toString)

View File

@ -88,6 +88,8 @@ object API {
.setConf("spark.driver.memory", "1g")
.setConf("spark.executor.memory", "1g")
.setConf("spark.cores.max", "2")
//.setConf("spark.checkpoint", PropertyUtil.getPropertyValue("checkpoint.path"))
.addFile(PropertyUtil.getConfigureFile())
.setMainClass("cn.piflow.api.StartFlowMain")
.addAppArgs(flowJson.stripMargin)
//.redirectOutput(stdout)

View File

@ -25,11 +25,11 @@ object StartFlowMain {
.appName(flowBean.name)
.enableHiveSupport()
.getOrCreate()
//val checkpointPath = spark.sparkContext.getConf.get("checkpoint.path")
val process = Runner.create()
.bind(classOf[SparkSession].getName, spark)
//.bind("checkpoint.path", PropertyUtil.getPropertyValue("checkpoint.path"))
.bind("checkpoint.path","hdfs://10.0.86.89:9000/xjzhu/piflow/checkpoints/")
.bind("checkpoint.path",PropertyUtil.getPropertyValue("checkpoint.path"))
.start(flow);
val applicationId = spark.sparkContext.applicationId
process.awaitTermination();

View File

@ -6,16 +6,21 @@ import java.util.Properties
object PropertyUtil {
private val prop: Properties = new Properties()
var fis: InputStream = null
try{
var path :String = ""
try{
//val path = Thread.currentThread().getContextClassLoader.getResource("config.properties").getPath
//fis = this.getClass.getResourceAsStream("")
val userDir = System.getProperty("user.dir")
val path = userDir + "/conf/" + "config.properties"
path = userDir + "/conf/" + "config.properties"
prop.load(new FileInputStream(path))
} catch{
case ex: Exception => ex.printStackTrace()
}
def getConfigureFile() : String = {
path
}
def getPropertyValue(propertyKey: String): String ={
val obj = prop.get(propertyKey)
if(obj != null){