select nested dataframe
This commit is contained in:
parent
0942318d80
commit
890e444141
|
@ -0,0 +1,50 @@
|
|||
{
|
||||
"flow":{
|
||||
"name":"xmlparserFlow",
|
||||
"uuid":"1234",
|
||||
"stops":[
|
||||
{
|
||||
"uuid":"0000",
|
||||
"name":"XmlParser",
|
||||
"bundle":"cn.piflow.bundle.xml.XmlParser",
|
||||
"properties":{
|
||||
"xmlpath":"hdfs://10.0.86.89:9000/xjzhu/cscd.xml",
|
||||
"rowTag":"paper"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid":"1111",
|
||||
"name":"SelectField",
|
||||
"bundle":"cn.piflow.bundle.common.SelectField",
|
||||
"properties":{
|
||||
"schema":"paper_id,title,authorlist.author.author_id,authorlist.author.author_name,authorlist.author.email"
|
||||
}
|
||||
|
||||
},
|
||||
{
|
||||
"uuid":"2222",
|
||||
"name":"JsonSave",
|
||||
"bundle":"cn.piflow.bundle.json.JsonSave",
|
||||
"properties":{
|
||||
"jsonSavePath":"hdfs://10.0.86.89:9000/xjzhu/cscd.json"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
],
|
||||
"paths":[
|
||||
{
|
||||
"from":"XmlParser",
|
||||
"outport":"",
|
||||
"inport":"",
|
||||
"to":"SelectField"
|
||||
},
|
||||
{
|
||||
"from":"SelectField",
|
||||
"outport":"",
|
||||
"inport":"",
|
||||
"to":"JsonSave"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
|
@ -29,6 +29,8 @@ class SelectField extends ConfigurableStop {
|
|||
|
||||
|
||||
var finalFieldDF : DataFrame = df.select(columnArray:_*)
|
||||
|
||||
finalFieldDF.printSchema()
|
||||
finalFieldDF.show(2)
|
||||
|
||||
out.write(finalFieldDF)
|
||||
|
|
Loading…
Reference in New Issue