yystopf
/
piflow
forked from opensci/piflow
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add next phase class 

`product distinct part`
`foreign key wash`
This commit is contained in:
coco11563 2019-07-26 16:40:25 +08:00
parent c62215120c
commit aab8b7854d
3 changed files with 47 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/HivePRDDistinct.scala
View File

@ -1,5 +1,25 @@
package cn.piflow.bundle.nsfc.distinct
class HivePRDDistinct {
import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
import cn.piflow.conf.{ConfigurableStop, PortEnum}
import cn.piflow.conf.bean.PropertyDescriptor
class HivePRDDistinct extends ConfigurableStop {
override val authorEmail: String = "xiaomeng7890@gmail.com"
override val description: String = ""
//
override val inportList: List[String] = List("productTable","productSpecTable")
override val outportList: List[String] = List("Relation", "Entity")
override def setProperties(map: Map[String, Any]): Unit = ???
override def getPropertyDescriptor(): List[PropertyDescriptor] = ???
override def getIcon(): Array[Byte] = ???
override def getGroup(): List[String] = ???
override def initialize(ctx: ProcessContext): Unit = ???
override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = ???
}

2
piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/HivePSNDistinct.scala
View File

@ -9,7 +9,7 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession}
import redis.clients.jedis.HostAndPort
class HivePSNDistinct extends ConfigurableStop{
class HivePSNDistinct extends ConfigurableStop {
override val authorEmail: String = "xiaomeng7890@gmail.com"
override val description: String = "person table distinct"
override val inportList: List[String] = List(PortEnum.DefaultPort.toString)

25
piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/foreignKeyWash.scala Normal file
View File

@ -0,0 +1,25 @@
package cn.piflow.bundle.nsfc.distinct
import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
import cn.piflow.conf.{ConfigurableStop, PortEnum}
import cn.piflow.conf.bean.PropertyDescriptor
class foreignKeyWash extends ConfigurableStop {
override val authorEmail: String = "xiaomeng7890@gmail.com"
override val description: String = ""
//
override val inportList: List[String] = List(PortEnum.DefaultPort.toString)
override val outportList: List[String] = List(PortEnum.DefaultPort.toString)
override def setProperties(map: Map[String, Any]): Unit = ???
override def getPropertyDescriptor(): List[PropertyDescriptor] = ???
override def getIcon(): Array[Byte] = ???
override def getGroup(): List[String] = ???
override def initialize(ctx: ProcessContext): Unit = ???
override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = ???
}