opensci
/
piflow
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

select nested dataframe

This commit is contained in:
judy0131 2018-10-17 14:34:22 +08:00
parent 0942318d80
commit 890e444141
2 changed files with 52 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
piflow-bundle/src/main/resources/xml.json Normal file
View File

@ -0,0 +1,50 @@
{
"flow":{
"name":"xmlparserFlow",
"uuid":"1234",
"stops":[
{
"uuid":"0000",
"name":"XmlParser",
"bundle":"cn.piflow.bundle.xml.XmlParser",
"properties":{
"xmlpath":"hdfs://10.0.86.89:9000/xjzhu/cscd.xml",
"rowTag":"paper"
}
},
{
"uuid":"1111",
"name":"SelectField",
"bundle":"cn.piflow.bundle.common.SelectField",
"properties":{
"schema":"paper_id,title,authorlist.author.author_id,authorlist.author.author_name,authorlist.author.email"
}
},
{
"uuid":"2222",
"name":"JsonSave",
"bundle":"cn.piflow.bundle.json.JsonSave",
"properties":{
"jsonSavePath":"hdfs://10.0.86.89:9000/xjzhu/cscd.json"
}
}
],
"paths":[
{
"from":"XmlParser",
"outport":"",
"inport":"",
"to":"SelectField"
},
{
"from":"SelectField",
"outport":"",
"inport":"",
"to":"JsonSave"
}
]
}
}

2
piflow-bundle/src/main/scala/cn/piflow/bundle/common/SelectField.scala
View File

@ -29,6 +29,8 @@ class SelectField extends ConfigurableStop {
var finalFieldDF : DataFrame = df.select(columnArray:_*) var finalFieldDF : DataFrame = df.select(columnArray:_*)
finalFieldDF.printSchema()
finalFieldDF.show(2) finalFieldDF.show(2)
out.write(finalFieldDF) out.write(finalFieldDF)