fix bug: can not find user defined Configurable Stop

piflow-bundle/pom.xml

@@ -16,11 +16,6 @@
             <artifactId>piflow-core</artifactId>
             <version>0.9</version>
         </dependency>
-        <!--<dependency>
-            <groupId>piflow</groupId>
-            <artifactId>piflow-conf</artifactId>
-            <version>0.9</version>
-        </dependency>-->
 
         <dependency>
             <groupId>org.clapper</groupId>

piflow-bundle/src/main/scala/cn/piflow/bundle/clean/EmailClean.scala

@@ -10,7 +10,7 @@ import cn.piflow.conf.util.MapUtil
 import org.apache.spark.sql.SparkSession
 
 class EmailClean extends ConfigurableStop{
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
   var columnName:String=_

piflow-bundle/src/main/scala/cn/piflow/bundle/clean/IdentityNumberClean.scala

@@ -14,7 +14,7 @@ import org.apache.spark.sql.SparkSession
 import scala.reflect.macros.ParseException
 
 class IdentityNumberClean extends ConfigurableStop{
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
   //var regex:String =_

piflow-bundle/src/main/scala/cn/piflow/bundle/clean/PhoneNumberClean.scala

@@ -8,7 +8,7 @@ import cn.piflow.conf.util.MapUtil
 import org.apache.spark.sql.SparkSession
 
 class PhoneNumberClean extends ConfigurableStop{
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
   var columnName:String=_

piflow-bundle/src/main/scala/cn/piflow/bundle/clean/TitleClean.scala

@@ -8,7 +8,7 @@ import cn.piflow.conf.util.MapUtil
 import org.apache.spark.sql.SparkSession
 
 class TitleClean extends ConfigurableStop{
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
   var columnName:String=_

piflow-bundle/src/main/scala/cn/piflow/bundle/common/Fork.scala

@@ -9,7 +9,7 @@ import scala.beans.BeanProperty
 
 class Fork extends ConfigurableStop{
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = -1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/common/Merge.scala

@@ -8,7 +8,7 @@ import scala.beans.BeanProperty
 
 class Merge extends ConfigurableStop{
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = -1
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/common/SelectField.scala

@@ -11,7 +11,7 @@ import scala.beans.BeanProperty
 
 class SelectField extends ConfigurableStop {
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/csv/CsvParser.scala

@@ -10,7 +10,7 @@ import org.apache.spark.sql.{DataFrame, SparkSession}
 
 class CsvParser extends ConfigurableStop{
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/csv/CsvSave.scala

@@ -7,9 +7,9 @@ import cn.piflow.conf.{ConfigurableStop, CsvGroup, StopGroup, StopGroupEnum}
 import org.apache.spark.sql.SaveMode
 
 class CsvSave extends ConfigurableStop{
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
-  override val inportCount: Int = 1
+  val inportCount: Int = 1
-  override val outportCount: Int = 0
+  val outportCount: Int = 0
 
   var csvSavePath: String = _
   var header: Boolean = _

piflow-bundle/src/main/scala/cn/piflow/bundle/file/FetchFile.scala

@@ -10,7 +10,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileSystem, Path}
 
 class FetchFile extends ConfigurableStop{
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
   var hdfs_path:String =_

piflow-bundle/src/main/scala/cn/piflow/bundle/file/PutFile.scala

@@ -13,7 +13,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.spark.sql.SparkSession
 
 class PutFile extends ConfigurableStop{
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
   var hdfs_path:String =_

piflow-bundle/src/main/scala/cn/piflow/bundle/file/RegexTextProcess.scala

@@ -9,7 +9,7 @@ import org.apache.spark.sql.SparkSession
 
 
 class RegexTextProcess extends ConfigurableStop{
-    override val authorEmail: String = "xiaoxiao@cnic.cn"
+    val authorEmail: String = "xiaoxiao@cnic.cn"
     val inportCount: Int = 0
     val outportCount: Int = 1
     var regex:String =_

piflow-bundle/src/main/scala/cn/piflow/bundle/ftp/LoadFromFtp.scala

@@ -13,7 +13,7 @@ import sun.net.ftp.{FtpClient, FtpDirEntry}
 import scala.reflect.io.Directory
 
 class LoadFromFtp extends ConfigurableStop{
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
   var url_str:String =_

piflow-bundle/src/main/scala/cn/piflow/bundle/ftp/UploadToFtp.scala

@@ -13,7 +13,7 @@ import sun.net.ftp.{FtpClient, FtpDirEntry}
 import scala.reflect.io.Directory
 
 class UploadToFtp extends ConfigurableStop{
-    override val authorEmail: String = "xiaoxiao@cnic.cn"
+    val authorEmail: String = "xiaoxiao@cnic.cn"
     val inportCount: Int = 0
     val outportCount: Int = 1
     var url_str:String =_

piflow-bundle/src/main/scala/cn/piflow/bundle/hive/PutHiveQL.scala

@@ -8,7 +8,7 @@ import org.apache.spark.sql.SparkSession
 
 class PutHiveQL extends ConfigurableStop {
 
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/hive/PutHiveStreaming.scala

@@ -10,7 +10,7 @@ import scala.beans.BeanProperty
 
 class PutHiveStreaming extends ConfigurableStop {
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = 0
 

piflow-bundle/src/main/scala/cn/piflow/bundle/hive/SelectHiveQL.scala

@@ -13,7 +13,7 @@ import scala.beans.BeanProperty
 
 class SelectHiveQL extends ConfigurableStop {
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/http/LoadZipFromUrl.scala

@@ -10,7 +10,7 @@ import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
 import org.apache.spark.sql.SparkSession
 
 class LoadZipFromUrl extends ConfigurableStop{
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
   var url_str:String =_

piflow-bundle/src/main/scala/cn/piflow/bundle/http/UnGZip.scala

@@ -11,7 +11,7 @@ import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveInp
 import org.apache.spark.sql.{DataFrame, SparkSession}
 
 class UnGZip extends ConfigurableStop {
-  override val authorEmail: String = "xiaoxiao@cnic.cn"
+  val authorEmail: String = "xiaoxiao@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
   val fileTypes:List[String]=List("tar.gz","tar","gz")

piflow-bundle/src/main/scala/cn/piflow/bundle/jdbc/JdbcRead.scala

@@ -9,7 +9,7 @@ import org.apache.spark.sql.SparkSession
 import scala.beans.BeanProperty
 
 class JdbcRead extends ConfigurableStop  {
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/jdbc/JdbcWrite.scala

@@ -12,7 +12,7 @@ import scala.beans.BeanProperty
 
 class JdbcWrite extends ConfigurableStop{
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = 0
 

piflow-bundle/src/main/scala/cn/piflow/bundle/json/JsonParser.scala

@@ -10,7 +10,7 @@ import scala.beans.BeanProperty
 
 class JsonPathParser extends ConfigurableStop{
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = 1
 
@@ -49,7 +49,7 @@ class JsonPathParser extends ConfigurableStop{
 }
 
 class JsonStringParser extends ConfigurableStop{
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/json/JsonSave.scala

@@ -10,7 +10,7 @@ import scala.beans.BeanProperty
 
 class JsonSave extends ConfigurableStop{
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = 0
 

piflow-bundle/src/main/scala/cn/piflow/bundle/script/DataFrameRowParser.scala

@@ -11,7 +11,7 @@ import scala.beans.BeanProperty
 
 class DataFrameRowParser extends ConfigurableStop{
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/script/ShellExecutor.scala

@@ -12,7 +12,7 @@ import sys.process._
 
 class ShellExecutor extends ConfigurableStop{
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 0
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/xml/FolderXmlParser.scala

@@ -17,9 +17,9 @@ import scala.util.control.Breaks._
   * Created by admin on 2018/8/27.
   */
 class FolderXmlParser extends ConfigurableStop{
-  override val authorEmail: String = "lijie"
+  val authorEmail: String = "lijie"
-  override val inportCount: Int = -1
+  val inportCount: Int = -1
-  override val outportCount: Int = 1
+  val outportCount: Int = 1
 
   var rowTag:String = _
   var xmlpath:String = _

piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlParser.scala

@@ -11,7 +11,7 @@ import scala.beans.BeanProperty
 
 class XmlParser extends ConfigurableStop {
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = 1
 

piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlSave.scala

@@ -10,7 +10,7 @@ import scala.beans.BeanProperty
 
 class XmlSave extends ConfigurableStop{
 
-  override val authorEmail: String = "xjzhu@cnic.cn"
+  val authorEmail: String = "xjzhu@cnic.cn"
   val inportCount: Int = 1
   val outportCount: Int = 0
 

piflow-bundle/src/main/scala/cn/piflow/conf/ConfigurableStop.scala

@@ -4,7 +4,7 @@ import cn.piflow.Stop
 import cn.piflow.conf.bean.PropertyDescriptor
 
 
-trait ConfigurableStop extends Stop{
+abstract class ConfigurableStop extends Stop{
 
   val authorEmail : String
   val inportCount : Int

piflow-bundle/src/main/scala/cn/piflow/conf/util/ClassUtil.scala

@@ -16,7 +16,7 @@ object ClassUtil {
   val configurableStopClass:String = "cn.piflow.conf.ConfigurableStop"
   //val classpath:String = "/opt/project/piflow/classpath"
 
-  /*def findAllConfigurableStop() : List[String] = {
+  /*def findAllConfigurableStopByClassFinder() : List[String] = {
 
     val classpath = System.getProperty("user.dir")
     var stopList : List[String] = List()
@@ -63,17 +63,29 @@ object ClassUtil {
     var stopList:List[ConfigurableStop] = List()
 
     val classpathFile = new File(classpath)
-    println("classpath is " + classpath)
+    //println("classpath is " + classpath)
     val finder = ClassFinder(getJarFile(classpathFile))
     val classes = finder.getClasses
-    val classMap = ClassFinder.classInfoMap(classes)
+    val it = classes.iterator
+
+    while(it.hasNext) {
+      val externalClass = it.next()
+      if(externalClass.superClassName.equals(configurableStopClass)){
+
+          val stopIntance = Class.forName(externalClass.name).newInstance()
+          stopList = stopIntance.asInstanceOf[ConfigurableStop] +: stopList
+      }
+    }
+    stopList
+
+    /*val classMap = ClassFinder.classInfoMap(classes)
     val plugins = ClassFinder.concreteSubclasses(configurableStopClass,classMap)
     plugins.foreach{
       pluginClassInfo =>
           val plugin = Class.forName(pluginClassInfo.name).newInstance()
           stopList = plugin.asInstanceOf[ConfigurableStop] +: stopList
     }
-    stopList
+    stopList*/
   }
 
   def findAllGroups() : List[String] = {
@@ -89,10 +101,22 @@ object ClassUtil {
     var stop:Option[ConfigurableStop] = None
 
     val classpathFile = new File(classpath)
-    println("classpath is " + classpath)
+    //println("classpath is " + classpath)
     val finder = ClassFinder(getJarFile(classpathFile))
     val classes = finder.getClasses
-    val classMap = ClassFinder.classInfoMap(classes)
+    val it = classes.iterator
+
+    while(it.hasNext) {
+      val externalClass = it.next()
+      if(externalClass.superClassName.equals(configurableStopClass)){
+        if (externalClass.name.equals(bundle)){
+          val stopIntance = Class.forName(externalClass.name).newInstance()
+          stop = Some(stopIntance.asInstanceOf[ConfigurableStop])
+          return stop
+        }
+      }
+    }
+    /*val classMap = ClassFinder.classInfoMap(classes)
     val plugins = ClassFinder.concreteSubclasses(configurableStopClass,classMap)
     plugins.foreach{
       pluginClassInfo =>
@@ -101,7 +125,7 @@ object ClassUtil {
           stop = Some(plugin.asInstanceOf[ConfigurableStop])
           return stop
         }
-    }
+    }*/
     stop
   }
 
@@ -153,9 +177,10 @@ object ClassUtil {
                 ("sensitive" -> property.sensitive.toString)) }) )
     val jsonString = compactRender(json)
     jsonString
+
   }
 
-  def main(args: Array[String]): Unit = {
+  /*def main(args: Array[String]): Unit = {
     //val stop = findConfigurableStop("cn.piflow.bundle.Class1")
     //val allConfigurableStopList = findAllConfigurableStop()
     /*val propertyDescriptorList = findConfigurableStopPropertyDescriptor("cn.piflow.bundle.Xjzhu")
@@ -166,13 +191,15 @@ object ClassUtil {
     val str = propertyJsonList.mkString(start, ",", end)
     println(str)*/
 
-    val stop = findConfigurableStopInClasspath("cn.piflow.bundle.Xjzhu")
+    /*val stop = findAllConfigurableStop()
-    val temp = 1
+    stop.foreach(s => println(s.getClass.getName))
-    //println(findAllGroups());
+    val temp = 1*/
 
-    //val stoplist = findAllGroups();
-    //println(stoplist)
 
-  }
+    /*val stop = findConfigurableStop("cn.piflow.bundle.Xjzhu")
+    println(stop.getClass.getName)*/
+
+
+  }*/
 
 }