opensci
/
piflow
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

This commit is contained in:
or 2020-04-02 18:35:54 +08:00
parent 6b4cabfb1f 11fb4168e3
commit 81c435c402
4 changed files with 60 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
piflow-bundle/src/main/resources/flow/neo4j/HiveToNeo4j.json
View File

@ -2,40 +2,45 @@
"flow":{
"name":"test",
"uuid":"1234",
"checkpoint":"Merge",
"stops":[
{
"uuid":"1111",
"name":"SelectHiveQL",
"bundle":"cn.piflow.bundle.hive.SelectHiveQL",
"properties":{
"hiveQL":"select * from test.user1"
}
},
{
"uuid":"0000",
"name":"HiveToNeo4j",
"bundle":"cn.piflow.bundle.neo4j.HiveToNeo4j",
"properties": {
"hiveQL":"select * from test.user1",
"hdfsDirPath": "/test",
"hdfsUrl":"hdfs://192.168.3.138:8020",
"fileName": "user1.csv",
"fileName": "user2.csv",
"delimiter":",",
"header": "true",
"url":"bolt://192.168.3.141:7687",
"userName": "neo4j",
"password":"null",
"cypher": "USING PERIODIC COMMIT 100 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user1.csv? op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})"
"cypher": "USING PERIODIC COMMIT 10 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user2.csv?op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})"
}
},
{
"uuid":"1335",
"name":"RunCypher",
"bundle":"cn.piflow.bundle.neo4j.RunCypher",
"properties": {
"url": "bolt://192.168.3.141:7687",
"userName": "neo4j",
"password": "",
"cql":"match(n:user) where n.userid ='11' set n.userclass =null"
}
}
],
"paths":[
{
"from":"SelectHiveQL",
"from":"HiveToNeo4j",
"outport":"",
"inport":"",
"to":"HiveToNeo4j"
"to":"RunCypher"
}
]

40
piflow-bundle/src/main/resources/flow/neo4j/RunCypher.json
View File

@ -2,35 +2,45 @@
"flow":{
"name":"test",
"uuid":"1234",
"checkpoint":"Merge",
"stops":[
{
"uuid":"1111",
"name":"SelectHiveQL",
"bundle":"cn.piflow.bundle.hive.SelectHiveQL",
"properties":{
"hiveQL":"select * from test.user1"
}
},
{
"uuid":"0000",
"name":"HiveToNeo4j",
"bundle":"cn.piflow.bundle.neo4j.HiveToNeo4j",
"properties": {
"url":"bolt://192.168.3.141:7687",
"userName": "neo4j",
"password":"null",
"cypher": "USING PERIODIC COMMIT 100 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user1.csv? op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})"
"hiveQL":"select * from test.user1",
"hdfsDirPath": "/test",
"hdfsUrl":"hdfs://192.168.3.138:8020",
"fileName": "user2.csv",
"delimiter":",",
"header": "true",
"url":"bolt://192.168.3.141:7687",
"userName": "neo4j",
"password":"null",
"cypher": "USING PERIODIC COMMIT 10 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user2.csv?op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})"
}
},
{
"uuid":"1335",
"name":"RunCypher",
"bundle":"cn.piflow.bundle.neo4j.RunCypher",
"properties": {
"url": "bolt://192.168.3.141:7687",
"userName": "neo4j",
"password": "",
"cql":"match(n:user) where n.userid ='11' set n.userclass =null"
}
}
],
"paths":[
{
"from":"SelectHiveQL",
"from":"HiveToNeo4j",
"outport":"",
"inport":"",
"to":"HiveToNeo4j"
"to":"RunCypher"
}
]

22
piflow-bundle/src/main/scala/cn/piflow/bundle/neo4j/HiveToNeo4j.scala
View File

@ -161,28 +161,28 @@ class HiveToNeo4j extends ConfigurableStop{
.displayName("HiveQL")
.defaultValue("")
.required(true)
.example("")
.example("select * from test.user1")
val hdfsDirPath = new PropertyDescriptor()
.name("hdfsDirPath")
.displayName("HdfsDirPath")
.defaultValue("/piflow-CSV-of-Neo4j/xxxxx")
.required(true)
.example("")
.example("/test")
val hdfsUrl = new PropertyDescriptor()
.name("hdfsUrl")
.displayName("HdfsUrl")
.defaultValue("hdfs://192.168.3.138:8020")
.required(true)
.example("")
.example("hdfs://192.168.3.138:8020")
val fileName = new PropertyDescriptor()
.name("fileName")
.displayName("FileName")
.defaultValue("")
.required(true)
.example("")
.example("test.csv")
val delimiter = new PropertyDescriptor()
.name("delimiter")
@ -190,7 +190,7 @@ class HiveToNeo4j extends ConfigurableStop{
.description("type is csv ,please set it ")
.defaultValue("¤")
.required(true)
.example("")
.example(",")
//header
val header = new PropertyDescriptor()
@ -200,15 +200,15 @@ class HiveToNeo4j extends ConfigurableStop{
.defaultValue("true")
.allowableValues(Set("true", "false"))
.required(true)
.example("")
.example("true")
val url=new PropertyDescriptor()
.name("url")
.displayName("Url")
.description("for example bolt://0.0.1.1:7687")
.description("The url of neo4j")
.defaultValue("bolt://127.0.0.1:7687")
.required(true)
.example("")
.example("bolt://127.0.0.1:7687")
val userName=new PropertyDescriptor()
.name("userName")
@ -216,7 +216,7 @@ class HiveToNeo4j extends ConfigurableStop{
.description("the user")
.defaultValue("neo4j")
.required(true)
.example("")
.example("neo4j")
val password=new PropertyDescriptor()
.name("password")
@ -224,7 +224,7 @@ class HiveToNeo4j extends ConfigurableStop{
.description("the password")
.defaultValue("")
.required(true)
.example("")
.example("123456")
val cypher=new PropertyDescriptor()
.name("cypher")
@ -232,7 +232,7 @@ class HiveToNeo4j extends ConfigurableStop{
.description(" the Cypher")
.defaultValue("")
.required(true)
.example("")
.example("USING PERIODIC COMMIT 10 LOAD CSV WITH HEADERS FROM 'http://192.168.3.138:50070/webhdfs/v1/test/user2.csv?op=OPEN' AS line FIELDTERMINATOR ',' CREATE (n:user{userid:line.id,username:line.name,userscore:line.score,userschool:line.school,userclass:line.class})")
descriptor = hiveQL :: descriptor

8
piflow-bundle/src/main/scala/cn/piflow/bundle/neo4j/RunCypher.scala
View File

@ -46,17 +46,19 @@ class RunCypher extends ConfigurableStop{
val url=new PropertyDescriptor().name("url")
.displayName("url")
.description("for example bolt://0.0.1.1:7687")
.description("The url of neo4j")
.defaultValue("")
.required(true)
.example("bolt://0.0.1.1:7687")
descriptor = url :: descriptor
val userName=new PropertyDescriptor()
.name("userName")
.displayName("UserName")
.description("The user")
.description("The user of neo4j")
.defaultValue("")
.required(true)
.example("neo4j")
descriptor = userName :: descriptor
val password=new PropertyDescriptor()
@ -66,6 +68,7 @@ class RunCypher extends ConfigurableStop{
.defaultValue("")
.required(true)
.sensitive(true)
.example("123456")
descriptor = password :: descriptor
val cql=new PropertyDescriptor()
@ -74,6 +77,7 @@ class RunCypher extends ConfigurableStop{
.description(" The Cypher")
.defaultValue("")
.required(true)
.example("match(n:user) where n.userid ='11' set n.userclass =5")
descriptor = cql :: descriptor
descriptor