yystopf
/
piflow
forked from opensci/piflow
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

update stop

This commit is contained in:
bao319 2020-04-02 18:02:35 +08:00
parent 03edd124c3
commit a2ce17289b
4 changed files with 60 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
piflow-bundle/src/main/resources/flow/neo4j/HiveToNeo4j.json
View File

@ -2,40 +2,45 @@
"flow":{ "flow":{
"name":"test", "name":"test",
"uuid":"1234", "uuid":"1234",
"checkpoint":"Merge",
"stops":[ "stops":[
{
"uuid":"1111",
"name":"SelectHiveQL",
"bundle":"cn.piflow.bundle.hive.SelectHiveQL",
"properties":{
"hiveQL":"select * from test.user1"
}
},
{ {
"uuid":"0000", "uuid":"0000",
"name":"HiveToNeo4j", "name":"HiveToNeo4j",
"bundle":"cn.piflow.bundle.neo4j.HiveToNeo4j", "bundle":"cn.piflow.bundle.neo4j.HiveToNeo4j",
"properties": { "properties": {
"hiveQL":"select * from test.user1",
"hdfsDirPath": "/test", "hdfsDirPath": "/test",
"hdfsUrl":"hdfs://192.168.3.138:8020", "hdfsUrl":"hdfs://192.168.3.138:8020",
"fileName": "user1.csv", "fileName": "user2.csv",
"delimiter":",", "delimiter":",",
"header": "true", "header": "true",
"url":"bolt://192.168.3.141:7687", "url":"bolt://192.168.3.141:7687",
"userName": "neo4j", "userName": "neo4j",
"password":"null", "password":"null",
"cypher": "USING PERIODIC COMMIT 100 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user1.csv? op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})" "cypher": "USING PERIODIC COMMIT 10 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user2.csv?op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})"
}
},
{
"uuid":"1335",
"name":"RunCypher",
"bundle":"cn.piflow.bundle.neo4j.RunCypher",
"properties": {
"url": "bolt://192.168.3.141:7687",
"userName": "neo4j",
"password": "",
"cql":"match(n:user) where n.userid ='11' set n.userclass =null"
} }
} }
], ],
"paths":[ "paths":[
{ {
"from":"SelectHiveQL", "from":"HiveToNeo4j",
"outport":"", "outport":"",
"inport":"", "inport":"",
"to":"HiveToNeo4j" "to":"RunCypher"
} }
] ]

40
piflow-bundle/src/main/resources/flow/neo4j/RunCypher.json
View File

@ -2,35 +2,45 @@
"flow":{ "flow":{
"name":"test", "name":"test",
"uuid":"1234", "uuid":"1234",
"checkpoint":"Merge",
"stops":[ "stops":[
{
"uuid":"1111",
"name":"SelectHiveQL",
"bundle":"cn.piflow.bundle.hive.SelectHiveQL",
"properties":{
"hiveQL":"select * from test.user1"
}
},
{ {
"uuid":"0000", "uuid":"0000",
"name":"HiveToNeo4j", "name":"HiveToNeo4j",
"bundle":"cn.piflow.bundle.neo4j.HiveToNeo4j", "bundle":"cn.piflow.bundle.neo4j.HiveToNeo4j",
"properties": { "properties": {
"url":"bolt://192.168.3.141:7687", "hiveQL":"select * from test.user1",
"userName": "neo4j", "hdfsDirPath": "/test",
"password":"null", "hdfsUrl":"hdfs://192.168.3.138:8020",
"cypher": "USING PERIODIC COMMIT 100 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user1.csv? op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})" "fileName": "user2.csv",
"delimiter":",",
"header": "true",
"url":"bolt://192.168.3.141:7687",
"userName": "neo4j",
"password":"null",
"cypher": "USING PERIODIC COMMIT 10 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user2.csv?op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})"
}
},
{
"uuid":"1335",
"name":"RunCypher",
"bundle":"cn.piflow.bundle.neo4j.RunCypher",
"properties": {
"url": "bolt://192.168.3.141:7687",
"userName": "neo4j",
"password": "",
"cql":"match(n:user) where n.userid ='11' set n.userclass =null"
} }
} }
], ],
"paths":[ "paths":[
{ {
"from":"SelectHiveQL", "from":"HiveToNeo4j",
"outport":"", "outport":"",
"inport":"", "inport":"",
"to":"HiveToNeo4j" "to":"RunCypher"
} }
] ]

22
piflow-bundle/src/main/scala/cn/piflow/bundle/neo4j/HiveToNeo4j.scala
View File

@ -161,28 +161,28 @@ class HiveToNeo4j extends ConfigurableStop{
.displayName("HiveQL") .displayName("HiveQL")
.defaultValue("") .defaultValue("")
.required(true) .required(true)
.example("") .example("select * from test.user1")
val hdfsDirPath = new PropertyDescriptor() val hdfsDirPath = new PropertyDescriptor()
.name("hdfsDirPath") .name("hdfsDirPath")
.displayName("HdfsDirPath") .displayName("HdfsDirPath")
.defaultValue("/piflow-CSV-of-Neo4j/xxxxx") .defaultValue("/piflow-CSV-of-Neo4j/xxxxx")
.required(true) .required(true)
.example("") .example("/test")
val hdfsUrl = new PropertyDescriptor() val hdfsUrl = new PropertyDescriptor()
.name("hdfsUrl") .name("hdfsUrl")
.displayName("HdfsUrl") .displayName("HdfsUrl")
.defaultValue("hdfs://192.168.3.138:8020") .defaultValue("hdfs://192.168.3.138:8020")
.required(true) .required(true)
.example("") .example("hdfs://192.168.3.138:8020")
val fileName = new PropertyDescriptor() val fileName = new PropertyDescriptor()
.name("fileName") .name("fileName")
.displayName("FileName") .displayName("FileName")
.defaultValue("") .defaultValue("")
.required(true) .required(true)
.example("") .example("test.csv")
val delimiter = new PropertyDescriptor() val delimiter = new PropertyDescriptor()
.name("delimiter") .name("delimiter")
@ -190,7 +190,7 @@ class HiveToNeo4j extends ConfigurableStop{
.description("type is csv ,please set it ") .description("type is csv ,please set it ")
.defaultValue("¤") .defaultValue("¤")
.required(true) .required(true)
.example("") .example(",")
//header //header
val header = new PropertyDescriptor() val header = new PropertyDescriptor()
@ -200,15 +200,15 @@ class HiveToNeo4j extends ConfigurableStop{
.defaultValue("true") .defaultValue("true")
.allowableValues(Set("true", "false")) .allowableValues(Set("true", "false"))
.required(true) .required(true)
.example("") .example("true")
val url=new PropertyDescriptor() val url=new PropertyDescriptor()
.name("url") .name("url")
.displayName("Url") .displayName("Url")
.description("for example bolt://0.0.1.1:7687") .description("The url of neo4j")
.defaultValue("bolt://127.0.0.1:7687") .defaultValue("bolt://127.0.0.1:7687")
.required(true) .required(true)
.example("") .example("bolt://127.0.0.1:7687")
val userName=new PropertyDescriptor() val userName=new PropertyDescriptor()
.name("userName") .name("userName")
@ -216,7 +216,7 @@ class HiveToNeo4j extends ConfigurableStop{
.description("the user") .description("the user")
.defaultValue("neo4j") .defaultValue("neo4j")
.required(true) .required(true)
.example("") .example("neo4j")
val password=new PropertyDescriptor() val password=new PropertyDescriptor()
.name("password") .name("password")
@ -224,7 +224,7 @@ class HiveToNeo4j extends ConfigurableStop{
.description("the password") .description("the password")
.defaultValue("") .defaultValue("")
.required(true) .required(true)
.example("") .example("123456")
val cypher=new PropertyDescriptor() val cypher=new PropertyDescriptor()
.name("cypher") .name("cypher")
@ -232,7 +232,7 @@ class HiveToNeo4j extends ConfigurableStop{
.description(" the Cypher") .description(" the Cypher")
.defaultValue("") .defaultValue("")
.required(true) .required(true)
.example("") .example("USING PERIODIC COMMIT 10 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user2.csv?op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})")
descriptor = hiveQL :: descriptor descriptor = hiveQL :: descriptor

8
piflow-bundle/src/main/scala/cn/piflow/bundle/neo4j/RunCypher.scala
View File

@ -46,17 +46,19 @@ class RunCypher extends ConfigurableStop{
val url=new PropertyDescriptor().name("url") val url=new PropertyDescriptor().name("url")
.displayName("url") .displayName("url")
.description("for example bolt://0.0.1.1:7687") .description("The url of neo4j")
.defaultValue("") .defaultValue("")
.required(true) .required(true)
.example("bolt://0.0.1.1:7687")
descriptor = url :: descriptor descriptor = url :: descriptor
val userName=new PropertyDescriptor() val userName=new PropertyDescriptor()
.name("userName") .name("userName")
.displayName("UserName") .displayName("UserName")
.description("The user") .description("The user of neo4j")
.defaultValue("") .defaultValue("")
.required(true) .required(true)
.example("neo4j")
descriptor = userName :: descriptor descriptor = userName :: descriptor
val password=new PropertyDescriptor() val password=new PropertyDescriptor()
@ -66,6 +68,7 @@ class RunCypher extends ConfigurableStop{
.defaultValue("") .defaultValue("")
.required(true) .required(true)
.sensitive(true) .sensitive(true)
.example("123456")
descriptor = password :: descriptor descriptor = password :: descriptor
val cql=new PropertyDescriptor() val cql=new PropertyDescriptor()
@ -74,6 +77,7 @@ class RunCypher extends ConfigurableStop{
.description(" The Cypher") .description(" The Cypher")
.defaultValue("") .defaultValue("")
.required(true) .required(true)
.example("match(n:user) where n.userid ='11' set n.userclass =5")
descriptor = cql :: descriptor descriptor = cql :: descriptor
descriptor descriptor