yystopf
/
piflow
forked from opensci/piflow
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bug fix

This commit is contained in:
coco11563 2019-07-26 09:57:44 +08:00
parent 01189e97aa
commit 6576e70280
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/HivePSNDistinct.scala
View File

@ -19,7 +19,7 @@ class HivePSNDistinct extends ConfigurableStop{
var tableName : String = _ //after wash
var sourceField : String = _
var timeField : String = _
var idKeys : String = _
var idKey : String = _
var noChange : Boolean = _
var distinctRule : String = _
var distinctFields : String = _
@ -90,7 +90,7 @@ class HivePSNDistinct extends ConfigurableStop{
row
}))
var processKeyArray = distinctFields.split(",")
processKeyArray += idKeys
processKeyArray +:= idKey
processKeyArray.foreach(key => { //对这里的每一组key
pairRDD = pairRDD.map(row => (cn.piflow.bundle.util.NSFCUtil.mkRowKey(inSchema, row._2, key), row)) //生成key pair， 若key不存在则生成UUID
.groupByKey