diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/HivePSNDistinct.scala b/piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/HivePSNDistinct.scala index 2552efc..23bf627 100644 --- a/piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/HivePSNDistinct.scala +++ b/piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/HivePSNDistinct.scala @@ -19,7 +19,7 @@ class HivePSNDistinct extends ConfigurableStop{ var tableName : String = _ //after wash var sourceField : String = _ var timeField : String = _ - var idKeys : String = _ + var idKey : String = _ var noChange : Boolean = _ var distinctRule : String = _ var distinctFields : String = _ @@ -90,7 +90,7 @@ class HivePSNDistinct extends ConfigurableStop{ row })) var processKeyArray = distinctFields.split(",") - processKeyArray += idKeys + processKeyArray +:= idKey processKeyArray.foreach(key => { //对这里的每一组key pairRDD = pairRDD.map(row => (cn.piflow.bundle.util.NSFCUtil.mkRowKey(inSchema, row._2, key), row)) //生成key pair, 若key不存在则生成UUID .groupByKey