forked from opensci/piflow
modify piflow-bin example
This commit is contained in:
judy0131
parent
f693c72b20
commit
e41a2aeb1e
|
|
@ -1,16 +1,11 @@
|
|||
#!/bin/bash
|
||||
if [ $PIFLOW_HOME -eq "" ];then
|
||||
echo "Please set PIFLOW_HOME, we need to read config.properties file from PIFLOW_HOME"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CONFIG_PATH="$PIFLOW_HOME/config.properties"
|
||||
SERVER_PATH="$PIFLOW_HOME/server.ip"
|
||||
SERVER_IP="$PIFLOW_HOME/server.ip"
|
||||
SERVER=""
|
||||
PORT=""
|
||||
|
||||
function server_info_init () {
|
||||
SERVER=`sed '/^server.ip=/!d;s/.*=//' $SERVER_PATH`
|
||||
SERVER=`sed '/^server.ip=/!d;s/.*=//' $SERVER_IP`
|
||||
PORT=`sed '/^server.port=/!d;s/.*=//' $CONFIG_PATH`
|
||||
}
|
||||
|
||||
|
|
@ -20,8 +15,8 @@ function help_info () {
|
|||
flow_help
|
||||
echo ""
|
||||
group_help
|
||||
#echo ""
|
||||
#project_help
|
||||
echo ""
|
||||
project_help
|
||||
}
|
||||
|
||||
function flow_help () {
|
||||
|
|
@ -127,20 +122,20 @@ function group_start () {
|
|||
echo "Group Start:"
|
||||
content=$(file_reader $1)
|
||||
echo "$content"
|
||||
curl_command="curl -0 -X POST http://$SERVER:$PORT/group/start -H \"Content-type: application/json\" -d '$content'"
|
||||
curl_command="curl -0 -X POST http://$SERVER:$PORT/flowGroup/start -H \"Content-type: application/json\" -d '$content'"
|
||||
eval $curl_command
|
||||
}
|
||||
|
||||
function group_info () {
|
||||
echo "Group Info:"
|
||||
curl_command="curl -Get http://$SERVER:$PORT/group/info?groupId=$1"
|
||||
curl_command="curl -Get http://$SERVER:$PORT/flowGroup/info?groupId=$1"
|
||||
echo $curl_command
|
||||
eval $curl_command
|
||||
}
|
||||
|
||||
function group_stop () {
|
||||
echo "Group Stop:"
|
||||
curl_command="curl -POST http://$SERVER:$PORT/group/stop -H \"Content-type: application/json\" -d '{\"groupId\":\"$1\"}'"
|
||||
curl_command="curl -POST http://$SERVER:$PORT/flowGroup/stop -H \"Content-type: application/json\" -d '{\"groupId\":\"$1\"}'"
|
||||
#echo $curl_command
|
||||
eval $curl_command
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,123 +1,154 @@
|
|||
{
|
||||
"flow": {
|
||||
"name": "test",
|
||||
"uuid": "1234",
|
||||
"checkpoint": "Merge",
|
||||
"stops": [{
|
||||
"uuid": "1111",
|
||||
"name": "XmlParser",
|
||||
"bundle": "cn.piflow.bundle.xml.XmlParser",
|
||||
"properties": {
|
||||
"xmlpath": "hdfs://10.0.86.191:9000/xjzhu/dblp.mini.xml",
|
||||
"rowTag": "phdthesis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "2222",
|
||||
"name": "SelectField",
|
||||
"bundle": "cn.piflow.bundle.common.SelectField",
|
||||
"properties": {
|
||||
"schema": "title,author,pages"
|
||||
}
|
||||
"flow": {
|
||||
"name": "Example",
|
||||
"executorMemory": "1g",
|
||||
"executorNumber": "1",
|
||||
"uuid": "8a80d63f720cdd2301723a4e679e2457",
|
||||
"paths": [
|
||||
{
|
||||
"inport": "",
|
||||
"from": "XmlParser",
|
||||
"to": "SelectField",
|
||||
"outport": ""
|
||||
},
|
||||
{
|
||||
"inport": "",
|
||||
"from": "Fork",
|
||||
"to": "CsvSave",
|
||||
"outport": "out1"
|
||||
},
|
||||
{
|
||||
"inport": "data2",
|
||||
"from": "SelectField",
|
||||
"to": "Merge",
|
||||
"outport": ""
|
||||
},
|
||||
{
|
||||
"inport": "",
|
||||
"from": "Merge",
|
||||
"to": "Fork",
|
||||
"outport": ""
|
||||
},
|
||||
{
|
||||
"inport": "data1",
|
||||
"from": "CsvParser",
|
||||
"to": "Merge",
|
||||
"outport": ""
|
||||
},
|
||||
{
|
||||
"inport": "",
|
||||
"from": "Fork",
|
||||
"to": "JsonSave",
|
||||
"outport": "out3"
|
||||
},
|
||||
{
|
||||
"inport": "",
|
||||
"from": "Fork",
|
||||
"to": "PutHiveMode",
|
||||
"outport": "out2"
|
||||
}
|
||||
],
|
||||
"executorCores": "1",
|
||||
"driverMemory": "1g",
|
||||
"stops": [
|
||||
{
|
||||
"name": "CsvSave",
|
||||
"bundle": "cn.piflow.bundle.csv.CsvSave",
|
||||
"uuid": "8a80d63f720cdd2301723a4e67a52467",
|
||||
"properties": {
|
||||
"csvSavePath": "hdfs://master:9000/xjzhu/phdthesis_result.csv",
|
||||
"partition": "",
|
||||
"header": "false",
|
||||
"saveMode": "append",
|
||||
"delimiter": ","
|
||||
},
|
||||
"customizedProperties": {
|
||||
|
||||
},
|
||||
{
|
||||
"uuid": "3333",
|
||||
"name": "PutHiveStreaming",
|
||||
"bundle": "cn.piflow.bundle.hive.PutHiveStreaming",
|
||||
"properties": {
|
||||
"database": "sparktest",
|
||||
"table": "dblp_phdthesis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "4444",
|
||||
"name": "CsvParser",
|
||||
"bundle": "cn.piflow.bundle.csv.CsvParser",
|
||||
"properties": {
|
||||
"csvPath": "hdfs://10.0.86.191:9000/xjzhu/phdthesis.csv",
|
||||
"header": "false",
|
||||
"delimiter": ",",
|
||||
"schema": "title,author,pages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "555",
|
||||
"name": "Merge",
|
||||
"bundle": "cn.piflow.bundle.common.Merge",
|
||||
"properties": {
|
||||
"inports": "data1,data2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "666",
|
||||
"name": "Fork",
|
||||
"bundle": "cn.piflow.bundle.common.Fork",
|
||||
"properties": {
|
||||
"outports": "out1,out2,out3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "777",
|
||||
"name": "JsonSave",
|
||||
"bundle": "cn.piflow.bundle.json.JsonSave",
|
||||
"properties": {
|
||||
"jsonSavePath": "hdfs://10.0.86.191:9000/xjzhu/phdthesis.json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uuid": "888",
|
||||
"name": "CsvSave",
|
||||
"bundle": "cn.piflow.bundle.csv.CsvSave",
|
||||
"properties": {
|
||||
"csvSavePath": "hdfs://10.0.86.191:9000/xjzhu/phdthesis_result.csv",
|
||||
"header": "true",
|
||||
"delimiter": ","
|
||||
}
|
||||
}
|
||||
],
|
||||
"paths": [{
|
||||
"from": "XmlParser",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "SelectField"
|
||||
},
|
||||
{
|
||||
"from": "SelectField",
|
||||
"outport": "",
|
||||
"inport": "data1",
|
||||
"to": "Merge"
|
||||
},
|
||||
{
|
||||
"from": "CsvParser",
|
||||
"outport": "",
|
||||
"inport": "data2",
|
||||
"to": "Merge"
|
||||
},
|
||||
{
|
||||
"from": "Merge",
|
||||
"outport": "",
|
||||
"inport": "",
|
||||
"to": "Fork"
|
||||
},
|
||||
{
|
||||
"from": "Fork",
|
||||
"outport": "out1",
|
||||
"inport": "",
|
||||
"to": "PutHiveStreaming"
|
||||
},
|
||||
{
|
||||
"from": "Fork",
|
||||
"outport": "out2",
|
||||
"inport": "",
|
||||
"to": "JsonSave"
|
||||
},
|
||||
{
|
||||
"from": "Fork",
|
||||
"outport": "out3",
|
||||
"inport": "",
|
||||
"to": "CsvSave"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "PutHiveMode",
|
||||
"bundle": "cn.piflow.bundle.hive.PutHiveMode",
|
||||
"uuid": "8a80d63f720cdd2301723a4e67a22461",
|
||||
"properties": {
|
||||
"database": "sparktest",
|
||||
"saveMode": "append",
|
||||
"table": "dblp_phdthesis"
|
||||
},
|
||||
"customizedProperties": {
|
||||
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "CsvParser",
|
||||
"bundle": "cn.piflow.bundle.csv.CsvParser",
|
||||
"uuid": "8a80d63f720cdd2301723a4e67a82470",
|
||||
"properties": {
|
||||
"schema": "title,author,pages",
|
||||
"csvPath": "hdfs://master:9000/xjzhu/phdthesis.csv",
|
||||
"delimiter": ",",
|
||||
"header": "false"
|
||||
},
|
||||
"customizedProperties": {
|
||||
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "JsonSave",
|
||||
"bundle": "cn.piflow.bundle.json.JsonSave",
|
||||
"uuid": "8a80d63f720cdd2301723a4e67a1245f",
|
||||
"properties": {
|
||||
"jsonSavePath": "hdfs://10.0.86.191:9000/xjzhu/phdthesis.json"
|
||||
},
|
||||
"customizedProperties": {
|
||||
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "XmlParser",
|
||||
"bundle": "cn.piflow.bundle.xml.XmlParser",
|
||||
"uuid": "8a80d63f720cdd2301723a4e67a7246d",
|
||||
"properties": {
|
||||
"rowTag": "phdthesis",
|
||||
"xmlpath": "hdfs://master:9000/xjzhu/dblp.mini.xml"
|
||||
},
|
||||
"customizedProperties": {
|
||||
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "SelectField",
|
||||
"bundle": "cn.piflow.bundle.common.SelectField",
|
||||
"uuid": "8a80d63f720cdd2301723a4e67aa2477",
|
||||
"properties": {
|
||||
"columnNames": "title,author,pages"
|
||||
},
|
||||
"customizedProperties": {
|
||||
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Merge",
|
||||
"bundle": "cn.piflow.bundle.common.Merge",
|
||||
"uuid": "8a80d63f720cdd2301723a4e67a92475",
|
||||
"properties": {
|
||||
"inports": "data1,data2"
|
||||
},
|
||||
"customizedProperties": {
|
||||
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Fork",
|
||||
"bundle": "cn.piflow.bundle.common.Fork",
|
||||
"uuid": "8a80d63f720cdd2301723a4e67a42465",
|
||||
"properties": {
|
||||
"outports": "out1,out3,out2"
|
||||
},
|
||||
"customizedProperties": {
|
||||
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
server.ip=10.0.85.83
|
||||
Loading…
Reference in New Issue