opensci
/
piflow
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

bug fix

This commit is contained in:
coco11563 2019-07-26 11:23:43 +08:00
parent e9ddb1d1b0
commit a76ad0b868
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/HivePSNDistinct.scala
View File

@ -25,7 +25,7 @@ class HivePSNDistinct extends ConfigurableStop{
var noChangeSource : String = _
var sourceField : String = _
var timeField : String = _
var idKeys : String = _
var idKey : String = _
var noChange : Boolean = _
var baseOnTime : Boolean = _
var baseOnField : Boolean = _
@ -98,7 +98,7 @@ class HivePSNDistinct extends ConfigurableStop{
row
}))
var processKeyArray = distinctFields.split(",")
processKeyArray += idKeys
processKeyArray +:= idKey
processKeyArray.foreach(key => { //对这里的每一组key
pairRDD = pairRDD.map(row => (cn.piflow.bundle.util.NSFCUtil.mkRowKey(inSchema, row._2, key), row)) //生成key pair， 若key不存在则生成UUID
.groupByKey