modify piflow-bin
This commit is contained in:
judy0131
parent
e41a2aeb1e
commit
40c380e089
|
|
@ -14,3 +14,6 @@
|
|||
piflow flowGroup start *.json
|
||||
piflow flowGroup stop groupId
|
||||
piflow flowGroup info groupId
|
||||
|
||||
4.please install piflow-web to configure flow in web page.
|
||||
https://github.com/cas-bigdatalab/piflow-web
|
||||
|
|
@ -122,65 +122,20 @@ function group_start () {
|
|||
echo "Group Start:"
|
||||
content=$(file_reader $1)
|
||||
echo "$content"
|
||||
curl_command="curl -0 -X POST http://$SERVER:$PORT/flowGroup/start -H \"Content-type: application/json\" -d '$content'"
|
||||
curl_command="curl -0 -X POST http://$SERVER:$PORT/group/start -H \"Content-type: application/json\" -d '$content'"
|
||||
eval $curl_command
|
||||
}
|
||||
|
||||
function group_info () {
|
||||
echo "Group Info:"
|
||||
curl_command="curl -Get http://$SERVER:$PORT/flowGroup/info?groupId=$1"
|
||||
curl_command="curl -Get http://$SERVER:$PORT/group/info?groupId=$1"
|
||||
echo $curl_command
|
||||
eval $curl_command
|
||||
}
|
||||
|
||||
function group_stop () {
|
||||
echo "Group Stop:"
|
||||
curl_command="curl -POST http://$SERVER:$PORT/flowGroup/stop -H \"Content-type: application/json\" -d '{\"groupId\":\"$1\"}'"
|
||||
#echo $curl_command
|
||||
eval $curl_command
|
||||
}
|
||||
|
||||
function project_process () {
|
||||
if [ $# -eq 2 ]
|
||||
then
|
||||
case "$1" in
|
||||
start)
|
||||
project_start $2
|
||||
;;
|
||||
stop)
|
||||
project_stop $2
|
||||
;;
|
||||
info)
|
||||
project_info $2
|
||||
;;
|
||||
*)
|
||||
group_help
|
||||
;;
|
||||
esac
|
||||
else
|
||||
echo $1
|
||||
group_help
|
||||
fi
|
||||
}
|
||||
|
||||
function project_start () {
|
||||
echo "Project Start:"
|
||||
content=$(file_reader $1)
|
||||
echo "$content"
|
||||
curl_command="curl -0 -X POST http://$SERVER:$PORT/project/start -H \"Content-type: application/json\" -d '$content'"
|
||||
eval $curl_command
|
||||
}
|
||||
|
||||
function project_info () {
|
||||
echo "Project Info:"
|
||||
curl_command="curl -Get http://$SERVER:$PORT/project/info?projectId=$1"
|
||||
#echo $curl_command
|
||||
eval $curl_command
|
||||
}
|
||||
|
||||
function project_stop () {
|
||||
echo "Project Stop:"
|
||||
curl_command="curl -POST http://$SERVER:$PORT/project/stop -H \"Content-type: application/json\" -d '{\"projectId\":\"$1\"}'"
|
||||
curl_command="curl -POST http://$SERVER:$PORT/group/stop -H \"Content-type: application/json\" -d '{\"groupId\":\"$1\"}'"
|
||||
#echo $curl_command
|
||||
eval $curl_command
|
||||
}
|
||||
|
|
@ -220,10 +175,6 @@ if [ $# -gt 0 ]; then
|
|||
#SERVICE=flowGroup
|
||||
echo "$(group_process $2 $3 2>/dev/null)"
|
||||
;;
|
||||
project)
|
||||
#SERVICE=project
|
||||
echo "$(project_process $2 $3 2>/dev/null)"
|
||||
;;
|
||||
*)
|
||||
help_info
|
||||
;;
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,283 +1,153 @@
|
|||
{
|
||||
"group":{
|
||||
"name": "Group",
|
||||
"uuid": "1111111111111",
|
||||
"groups": [
|
||||
{"group": {
|
||||
"name": "Group1",
|
||||
"uuid": "1111111111111",
|
||||
|
||||
"flows": [
|
||||
{"flow": {
|
||||
"name": "flow1",
|
||||
"uuid": "1234",
|
||||
"executorNumber": "2",
|
||||
"executorMemory": "1g",
|
||||
"executorCores": "1",
|
||||
"stops": [{
|
||||
"uuid": "1111",
|
||||
"name": "XmlParser",
|
||||
"bundle": "cn.piflow.bundle.xml.XmlParser",
|
||||
"properties": {
|
||||
"xmlpath": "hdfs://10.0.86.89:9000/xjzhu/dblp.mini.xml",
|
||||
"rowTag": "phdthesis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "2222",
|
||||
"name": "SelectField",
|
||||
"bundle": "cn.piflow.bundle.common.SelectField",
|
||||
"properties": {
|
||||
"schema": "title,author,pages"
|
||||
}
|
||||
|
||||
},
|
||||
{
|
||||
"uuid": "3333",
|
||||
"name": "PutHiveStreaming",
|
||||
"bundle": "cn.piflow.bundle.hive.PutHiveStreaming",
|
||||
"properties": {
|
||||
"database": "sparktest",
|
||||
"table": "dblp_phdthesis"
|
||||
}
|
||||
}
|
||||
],
|
||||
"paths": [{
|
||||
"from": "XmlParser",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "SelectField"
|
||||
},
|
||||
{
|
||||
"from": "SelectField",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "PutHiveStreaming"
|
||||
}
|
||||
]
|
||||
}},
|
||||
{"flow": {
|
||||
"name": "flow2",
|
||||
"uuid": "5678",
|
||||
"stops": [{
|
||||
"uuid": "1111",
|
||||
"name": "XmlParser",
|
||||
"bundle": "cn.piflow.bundle.xml.XmlParser",
|
||||
"properties": {
|
||||
"xmlpath": "hdfs://10.0.86.89:9000/xjzhu/dblp.mini.xml",
|
||||
"rowTag": "phdthesis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "2222",
|
||||
"name": "SelectField",
|
||||
"bundle": "cn.piflow.bundle.common.SelectField",
|
||||
"properties": {
|
||||
"schema": "title,author,pages"
|
||||
}
|
||||
|
||||
},
|
||||
{
|
||||
"uuid": "3333",
|
||||
"name": "PutHiveStreaming",
|
||||
"bundle": "cn.piflow.bundle.hive.PutHiveStreaming",
|
||||
"properties": {
|
||||
"database": "sparktest",
|
||||
"table": "dblp_phdthesis"
|
||||
}
|
||||
}
|
||||
],
|
||||
"paths": [{
|
||||
"from": "XmlParser",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "SelectField"
|
||||
},
|
||||
{
|
||||
"from": "SelectField",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "PutHiveStreaming"
|
||||
}
|
||||
]
|
||||
}}
|
||||
],
|
||||
|
||||
"conditions": [{
|
||||
"entry": "flow2",
|
||||
"after": "flow1"
|
||||
}]
|
||||
}},
|
||||
{"group": {
|
||||
"name": "Group3",
|
||||
"uuid": "1111111111111",
|
||||
|
||||
"flows": [
|
||||
{"flow": {
|
||||
"name": "flow4",
|
||||
"uuid": "1234",
|
||||
"executorNumber": "2",
|
||||
"executorMemory": "1g",
|
||||
"executorCores": "1",
|
||||
"stops": [{
|
||||
"uuid": "1111",
|
||||
"name": "XmlParser",
|
||||
"bundle": "cn.piflow.bundle.xml.XmlParser",
|
||||
"properties": {
|
||||
"xmlpath": "hdfs://10.0.86.89:9000/xjzhu/dblp.mini.xml",
|
||||
"rowTag": "phdthesis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "2222",
|
||||
"name": "SelectField",
|
||||
"bundle": "cn.piflow.bundle.common.SelectField",
|
||||
"properties": {
|
||||
"schema": "title,author,pages"
|
||||
}
|
||||
|
||||
},
|
||||
{
|
||||
"uuid": "3333",
|
||||
"name": "PutHiveStreaming",
|
||||
"bundle": "cn.piflow.bundle.hive.PutHiveStreaming",
|
||||
"properties": {
|
||||
"database": "sparktest",
|
||||
"table": "dblp_phdthesis"
|
||||
}
|
||||
}
|
||||
],
|
||||
"paths": [{
|
||||
"from": "XmlParser",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "SelectField"
|
||||
},
|
||||
{
|
||||
"from": "SelectField",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "PutHiveStreaming"
|
||||
}
|
||||
]
|
||||
}},
|
||||
{"flow": {
|
||||
"name": "flow5",
|
||||
"uuid": "5678",
|
||||
"stops": [{
|
||||
"uuid": "1111",
|
||||
"name": "XmlParser",
|
||||
"bundle": "cn.piflow.bundle.xml.XmlParser",
|
||||
"properties": {
|
||||
"xmlpath": "hdfs://10.0.86.89:9000/xjzhu/dblp.mini.xml",
|
||||
"rowTag": "phdthesis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "2222",
|
||||
"name": "SelectField",
|
||||
"bundle": "cn.piflow.bundle.common.SelectField",
|
||||
"properties": {
|
||||
"schema": "title,author,pages"
|
||||
}
|
||||
|
||||
},
|
||||
{
|
||||
"uuid": "3333",
|
||||
"name": "PutHiveStreaming",
|
||||
"bundle": "cn.piflow.bundle.hive.PutHiveStreaming",
|
||||
"properties": {
|
||||
"database": "sparktest",
|
||||
"table": "dblp_phdthesis"
|
||||
}
|
||||
}
|
||||
],
|
||||
"paths": [{
|
||||
"from": "XmlParser",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "SelectField"
|
||||
},
|
||||
{
|
||||
"from": "SelectField",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "PutHiveStreaming"
|
||||
}
|
||||
]
|
||||
}}
|
||||
],
|
||||
|
||||
|
||||
"conditions": [{
|
||||
"entry": "flow5",
|
||||
"after": "flow4"
|
||||
}]
|
||||
}}
|
||||
],
|
||||
|
||||
"flows":[
|
||||
{"flow": {
|
||||
"name": "flow3",
|
||||
"uuid": "91011",
|
||||
"executorNumber": "2",
|
||||
"executorMemory": "1g",
|
||||
"executorCores": "1",
|
||||
"stops": [{
|
||||
"uuid": "1111",
|
||||
"name": "XmlParser",
|
||||
"bundle": "cn.piflow.bundle.xml.XmlParser",
|
||||
"properties": {
|
||||
"xmlpath": "hdfs://10.0.86.89:9000/xjzhu/dblp.mini.xml",
|
||||
"rowTag": "phdthesis"
|
||||
"group" : {
|
||||
"flows" : [ {
|
||||
"flow" : {
|
||||
"executorNumber" : "1",
|
||||
"driverMemory" : "1g",
|
||||
"executorMemory" : "1g",
|
||||
"executorCores" : "1",
|
||||
"paths" : [ {
|
||||
"inport" : "",
|
||||
"from" : "MockData",
|
||||
"to" : "ShowData",
|
||||
"outport" : ""
|
||||
} ],
|
||||
"name" : "f4",
|
||||
"stops" : [ {
|
||||
"customizedProperties" : { },
|
||||
"name" : "MockData",
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e825a6",
|
||||
"bundle" : "cn.piflow.bundle.common.MockData",
|
||||
"properties" : {
|
||||
"schema" : "title:String, author:String, age:Int",
|
||||
"count" : "10"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "2222",
|
||||
"name": "SelectField",
|
||||
"bundle": "cn.piflow.bundle.common.SelectField",
|
||||
"properties": {
|
||||
"schema": "title,author,pages"
|
||||
}
|
||||
|
||||
},
|
||||
{
|
||||
"uuid": "3333",
|
||||
"name": "PutHiveStreaming",
|
||||
"bundle": "cn.piflow.bundle.hive.PutHiveStreaming",
|
||||
"properties": {
|
||||
"database": "sparktest",
|
||||
"table": "dblp_phdthesis"
|
||||
}
|
||||
}, {
|
||||
"customizedProperties" : { },
|
||||
"name" : "ShowData",
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e825a4",
|
||||
"bundle" : "cn.piflow.bundle.external.ShowData",
|
||||
"properties" : {
|
||||
"showNumber" : "5"
|
||||
}
|
||||
],
|
||||
"paths": [{
|
||||
"from": "XmlParser",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "SelectField"
|
||||
},
|
||||
{
|
||||
"from": "SelectField",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "PutHiveStreaming"
|
||||
}
|
||||
]
|
||||
}}
|
||||
],
|
||||
|
||||
"conditions": [
|
||||
{
|
||||
"entry": "flow3",
|
||||
"after": "Group1"
|
||||
},
|
||||
{
|
||||
"entry": "Group3",
|
||||
"after": "flow3"
|
||||
} ],
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e825a2"
|
||||
}
|
||||
]
|
||||
}, {
|
||||
"flow" : {
|
||||
"executorNumber" : "1",
|
||||
"driverMemory" : "1g",
|
||||
"executorMemory" : "1g",
|
||||
"executorCores" : "1",
|
||||
"paths" : [ {
|
||||
"inport" : "",
|
||||
"from" : "MockData",
|
||||
"to" : "ShowData",
|
||||
"outport" : ""
|
||||
} ],
|
||||
"name" : "f3",
|
||||
"stops" : [ {
|
||||
"customizedProperties" : { },
|
||||
"name" : "MockData",
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9ea25ba",
|
||||
"bundle" : "cn.piflow.bundle.common.MockData",
|
||||
"properties" : {
|
||||
"schema" : "title:String, author:String, age:Int",
|
||||
"count" : "10"
|
||||
}
|
||||
}, {
|
||||
"customizedProperties" : { },
|
||||
"name" : "ShowData",
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9ea25b8",
|
||||
"bundle" : "cn.piflow.bundle.external.ShowData",
|
||||
"properties" : {
|
||||
"showNumber" : "5"
|
||||
}
|
||||
} ],
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e925b6"
|
||||
}
|
||||
} ],
|
||||
"name" : "SimpleGroup",
|
||||
"groups" : [ {
|
||||
"group" : {
|
||||
"flows" : [ {
|
||||
"flow" : {
|
||||
"executorNumber" : "1",
|
||||
"driverMemory" : "1g",
|
||||
"executorMemory" : "1g",
|
||||
"executorCores" : "1",
|
||||
"paths" : [ {
|
||||
"inport" : "",
|
||||
"from" : "MockData",
|
||||
"to" : "ShowData",
|
||||
"outport" : ""
|
||||
} ],
|
||||
"name" : "MockData",
|
||||
"stops" : [ {
|
||||
"customizedProperties" : { },
|
||||
"name" : "MockData",
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e42577",
|
||||
"bundle" : "cn.piflow.bundle.common.MockData",
|
||||
"properties" : {
|
||||
"schema" : "title:String, author:String, age:Int",
|
||||
"count" : "10"
|
||||
}
|
||||
}, {
|
||||
"customizedProperties" : { },
|
||||
"name" : "ShowData",
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e42575",
|
||||
"bundle" : "cn.piflow.bundle.external.ShowData",
|
||||
"properties" : {
|
||||
"showNumber" : "5"
|
||||
}
|
||||
} ],
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e32573"
|
||||
}
|
||||
}, {
|
||||
"flow" : {
|
||||
"executorNumber" : "1",
|
||||
"driverMemory" : "1g",
|
||||
"executorMemory" : "1g",
|
||||
"executorCores" : "1",
|
||||
"paths" : [ {
|
||||
"inport" : "",
|
||||
"from" : "MockData",
|
||||
"to" : "ShowData",
|
||||
"outport" : ""
|
||||
} ],
|
||||
"name" : "MockData",
|
||||
"stops" : [ {
|
||||
"customizedProperties" : { },
|
||||
"name" : "MockData",
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e6258b",
|
||||
"bundle" : "cn.piflow.bundle.common.MockData",
|
||||
"properties" : {
|
||||
"schema" : "title:String, author:String, age:Int",
|
||||
"count" : "10"
|
||||
}
|
||||
}, {
|
||||
"customizedProperties" : { },
|
||||
"name" : "ShowData",
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e62589",
|
||||
"bundle" : "cn.piflow.bundle.external.ShowData",
|
||||
"properties" : {
|
||||
"showNumber" : "5"
|
||||
}
|
||||
} ],
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e52587"
|
||||
}
|
||||
} ],
|
||||
"name" : "g1",
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e32572"
|
||||
}
|
||||
} ],
|
||||
"conditions" : [ {
|
||||
"entry" : "f4",
|
||||
"after" : "g1"
|
||||
}, {
|
||||
"entry" : "f3",
|
||||
"after" : "g1"
|
||||
} ],
|
||||
"uuid" : "8a80d63f720cdd2301723ad9f9e22571"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue