modify flow example
This commit is contained in:
judy0131
parent
73ebc17e57
commit
5a4d0440b7
51
README.md
51
README.md
|
|
@ -38,7 +38,6 @@ Try PiFlow v0.6 with: http://piflow.cstcloud.cn/piflow-web/
|
||||||
* JDK 1.8
|
* JDK 1.8
|
||||||
* Scala-2.11.8
|
* Scala-2.11.8
|
||||||
* Apache Maven 3.1.0 or newer
|
* Apache Maven 3.1.0 or newer
|
||||||
* Git Client (used during build process by 'bower' plugin)
|
|
||||||
* Spark-2.1.0、 Spark-2.2.0、 Spark-2.3.0
|
* Spark-2.1.0、 Spark-2.2.0、 Spark-2.3.0
|
||||||
* Hadoop-2.6.0
|
* Hadoop-2.6.0
|
||||||
|
|
||||||
|
|
@ -47,10 +46,10 @@ Try PiFlow v0.6 with: http://piflow.cstcloud.cn/piflow-web/
|
||||||
### To Build:
|
### To Build:
|
||||||
- `install external package`
|
- `install external package`
|
||||||
|
|
||||||
mvn install:install-file -Dfile=/.../piflow/piflow-bundle/lib/spark-xml_2.11-0.4.2.jar -DgroupId=com.databricks -DartifactId=spark-xml_2.11 -Dversion=0.4.2 -Dpackaging=jar
|
mvn install:install-file -Dfile=/../piflow/piflow-bundle/lib/spark-xml_2.11-0.4.2.jar -DgroupId=com.databricks -DartifactId=spark-xml_2.11 -Dversion=0.4.2 -Dpackaging=jar
|
||||||
mvn install:install-file -Dfile=/.../piflow/piflow-bundle/lib/java_memcached-release_2.6.6.jar -DgroupId=com.memcached -DartifactId=java_memcached-release -Dversion=2.6.6 -Dpackaging=jar
|
mvn install:install-file -Dfile=/../piflow/piflow-bundle/lib/java_memcached-release_2.6.6.jar -DgroupId=com.memcached -DartifactId=java_memcached-release -Dversion=2.6.6 -Dpackaging=jar
|
||||||
mvn install:install-file -Dfile=/.../piflow/piflow-bundle/lib/ojdbc6-11.2.0.3.jar -DgroupId=oracle -DartifactId=ojdbc6 -Dversion=11.2.0.3 -Dpackaging=jar
|
mvn install:install-file -Dfile=/../piflow/piflow-bundle/lib/ojdbc6-11.2.0.3.jar -DgroupId=oracle -DartifactId=ojdbc6 -Dversion=11.2.0.3 -Dpackaging=jar
|
||||||
mvn install:install-file -Dfile=/.../piflow/piflow-bundle/lib/edtftpj.jar -DgroupId=ftpClient -DartifactId=edtftp -Dversion=1.0.0 -Dpackaging=jar
|
mvn install:install-file -Dfile=/../piflow/piflow-bundle/lib/edtftpj.jar -DgroupId=ftpClient -DartifactId=edtftp -Dversion=1.0.0 -Dpackaging=jar
|
||||||
|
|
||||||
|
|
||||||
- `mvn clean package -Dmaven.test.skip=true`
|
- `mvn clean package -Dmaven.test.skip=true`
|
||||||
|
|
@ -73,10 +72,27 @@ Try PiFlow v0.6 with: http://piflow.cstcloud.cn/piflow-web/
|
||||||
|
|
||||||
### Run Piflow Server：
|
### Run Piflow Server：
|
||||||
|
|
||||||
- `run piflow server on intellij`:
|
- `run piflow server on Intellij`:
|
||||||
- edit config.properties
|
- download piflow: git clone https://github.com/cas-bigdatalab/piflow.git
|
||||||
- build piflow to generate piflow-server-0.9.jar
|
- import piflow into Intellij
|
||||||
- main class is cn.piflow.api.Main(remember to set SPARK_HOME)
|
- edit config.properties file
|
||||||
|
- build piflow to generate piflow jar:
|
||||||
|
- Edit Configurations --> Add New Configuration --> Maven
|
||||||
|
- Name: package
|
||||||
|
- Command line: clean package -Dmaven.test.skip=true -X
|
||||||
|
- run 'package' (piflow jar file will be built in ../piflow/piflow-server/target/piflow-server-0.9.jar)
|
||||||
|
|
||||||
|
- run HttpService:
|
||||||
|
- Edit Configurations --> Add New Configuration --> Application
|
||||||
|
- Name: HttpService
|
||||||
|
- Main class : cn.piflow.api.Main
|
||||||
|
- Environment Variable: SPARK_HOME=/opt/spark-2.2.0-bin-hadoop2.6(change the path to your spark home)
|
||||||
|
- run 'HttpService'
|
||||||
|
|
||||||
|
- test HttpService:
|
||||||
|
- run /../piflow/piflow-server/src/main/scala/cn/piflow/api/HTTPClientStartMockDataFlow.scala
|
||||||
|
- change the piflow server ip and port to your configure
|
||||||
|
|
||||||
|
|
||||||
- `run piflow server by release version`:
|
- `run piflow server by release version`:
|
||||||
- download piflow.tar.gz:
|
- download piflow.tar.gz:
|
||||||
|
|
@ -88,7 +104,24 @@ Try PiFlow v0.6 with: http://piflow.cstcloud.cn/piflow-web/
|
||||||
tar -zxvf piflow.tar.gz
|
tar -zxvf piflow.tar.gz
|
||||||
|
|
||||||
- edit config.properties
|
- edit config.properties
|
||||||
|
|
||||||
- run start.sh、stop.sh、 restart.sh、 status.sh
|
- run start.sh、stop.sh、 restart.sh、 status.sh
|
||||||
|
|
||||||
|
- test piflow server
|
||||||
|
- set PIFLOW_HOME
|
||||||
|
- vim /etc/profile
|
||||||
|
export PIFLOW_HOME=/yourPiflowPath/bin
|
||||||
|
export PATH=$PATH:$PIFLOW_HOME/bin
|
||||||
|
- command
|
||||||
|
piflow flow start example/mockDataFlow.json
|
||||||
|
piflow flow stop appID
|
||||||
|
piflow flow info appID
|
||||||
|
piflow flow log appID
|
||||||
|
|
||||||
|
piflow flowGroup start example/mockDataGroup.json
|
||||||
|
piflow flowGroup stop groupId
|
||||||
|
piflow flowGroup info groupId
|
||||||
|
|
||||||
- `how to configure config.properties`
|
- `how to configure config.properties`
|
||||||
|
|
||||||
#spark and yarn config
|
#spark and yarn config
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
{
|
||||||
|
"flow": {
|
||||||
|
"name": "MockData",
|
||||||
|
"executorMemory": "1g",
|
||||||
|
"executorNumber": "1",
|
||||||
|
"uuid": "8a80d63f720cdd2301723b7461d92600",
|
||||||
|
"paths": [
|
||||||
|
{
|
||||||
|
"inport": "",
|
||||||
|
"from": "MockData",
|
||||||
|
"to": "ShowData",
|
||||||
|
"outport": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"executorCores": "1",
|
||||||
|
"driverMemory": "1g",
|
||||||
|
"stops": [
|
||||||
|
{
|
||||||
|
"name": "MockData",
|
||||||
|
"bundle": "cn.piflow.bundle.common.MockData",
|
||||||
|
"uuid": "8a80d63f720cdd2301723b7461d92604",
|
||||||
|
"properties": {
|
||||||
|
"schema": "title:String, author:String, age:Int",
|
||||||
|
"count": "10"
|
||||||
|
},
|
||||||
|
"customizedProperties": {
|
||||||
|
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ShowData",
|
||||||
|
"bundle": "cn.piflow.bundle.external.ShowData",
|
||||||
|
"uuid": "8a80d63f720cdd2301723b7461d92602",
|
||||||
|
"properties": {
|
||||||
|
"showNumber": "5"
|
||||||
|
},
|
||||||
|
"customizedProperties": {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,153 @@
|
||||||
|
{
|
||||||
|
"group" : {
|
||||||
|
"flows" : [ {
|
||||||
|
"flow" : {
|
||||||
|
"executorNumber" : "1",
|
||||||
|
"driverMemory" : "1g",
|
||||||
|
"executorMemory" : "1g",
|
||||||
|
"executorCores" : "1",
|
||||||
|
"paths" : [ {
|
||||||
|
"inport" : "",
|
||||||
|
"from" : "MockData",
|
||||||
|
"to" : "ShowData",
|
||||||
|
"outport" : ""
|
||||||
|
} ],
|
||||||
|
"name" : "f4",
|
||||||
|
"stops" : [ {
|
||||||
|
"customizedProperties" : { },
|
||||||
|
"name" : "MockData",
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b72649",
|
||||||
|
"bundle" : "cn.piflow.bundle.common.MockData",
|
||||||
|
"properties" : {
|
||||||
|
"schema" : "title:String, author:String, age:Int",
|
||||||
|
"count" : "10"
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
"customizedProperties" : { },
|
||||||
|
"name" : "ShowData",
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b72647",
|
||||||
|
"bundle" : "cn.piflow.bundle.external.ShowData",
|
||||||
|
"properties" : {
|
||||||
|
"showNumber" : "5"
|
||||||
|
}
|
||||||
|
} ],
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b62645"
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
"flow" : {
|
||||||
|
"executorNumber" : "1",
|
||||||
|
"driverMemory" : "1g",
|
||||||
|
"executorMemory" : "1g",
|
||||||
|
"executorCores" : "1",
|
||||||
|
"paths" : [ {
|
||||||
|
"inport" : "",
|
||||||
|
"from" : "MockData",
|
||||||
|
"to" : "ShowData",
|
||||||
|
"outport" : ""
|
||||||
|
} ],
|
||||||
|
"name" : "f3",
|
||||||
|
"stops" : [ {
|
||||||
|
"customizedProperties" : { },
|
||||||
|
"name" : "MockData",
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b9265d",
|
||||||
|
"bundle" : "cn.piflow.bundle.common.MockData",
|
||||||
|
"properties" : {
|
||||||
|
"schema" : "title:String, author:String, age:Int",
|
||||||
|
"count" : "10"
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
"customizedProperties" : { },
|
||||||
|
"name" : "ShowData",
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b9265b",
|
||||||
|
"bundle" : "cn.piflow.bundle.external.ShowData",
|
||||||
|
"properties" : {
|
||||||
|
"showNumber" : "5"
|
||||||
|
}
|
||||||
|
} ],
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b82659"
|
||||||
|
}
|
||||||
|
} ],
|
||||||
|
"name" : "SimpleGroup",
|
||||||
|
"groups" : [ {
|
||||||
|
"group" : {
|
||||||
|
"flows" : [ {
|
||||||
|
"flow" : {
|
||||||
|
"executorNumber" : "1",
|
||||||
|
"driverMemory" : "1g",
|
||||||
|
"executorMemory" : "1g",
|
||||||
|
"executorCores" : "1",
|
||||||
|
"paths" : [ {
|
||||||
|
"inport" : "",
|
||||||
|
"from" : "MockData",
|
||||||
|
"to" : "ShowData",
|
||||||
|
"outport" : ""
|
||||||
|
} ],
|
||||||
|
"name" : "MockData",
|
||||||
|
"stops" : [ {
|
||||||
|
"customizedProperties" : { },
|
||||||
|
"name" : "MockData",
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b4261a",
|
||||||
|
"bundle" : "cn.piflow.bundle.common.MockData",
|
||||||
|
"properties" : {
|
||||||
|
"schema" : "title:String, author:String, age:Int",
|
||||||
|
"count" : "10"
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
"customizedProperties" : { },
|
||||||
|
"name" : "ShowData",
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b32618",
|
||||||
|
"bundle" : "cn.piflow.bundle.external.ShowData",
|
||||||
|
"properties" : {
|
||||||
|
"showNumber" : "5"
|
||||||
|
}
|
||||||
|
} ],
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b32616"
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
"flow" : {
|
||||||
|
"executorNumber" : "1",
|
||||||
|
"driverMemory" : "1g",
|
||||||
|
"executorMemory" : "1g",
|
||||||
|
"executorCores" : "1",
|
||||||
|
"paths" : [ {
|
||||||
|
"inport" : "",
|
||||||
|
"from" : "MockData",
|
||||||
|
"to" : "ShowData",
|
||||||
|
"outport" : ""
|
||||||
|
} ],
|
||||||
|
"name" : "MockData",
|
||||||
|
"stops" : [ {
|
||||||
|
"customizedProperties" : { },
|
||||||
|
"name" : "MockData",
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b5262e",
|
||||||
|
"bundle" : "cn.piflow.bundle.common.MockData",
|
||||||
|
"properties" : {
|
||||||
|
"schema" : "title:String, author:String, age:Int",
|
||||||
|
"count" : "10"
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
"customizedProperties" : { },
|
||||||
|
"name" : "ShowData",
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b5262c",
|
||||||
|
"bundle" : "cn.piflow.bundle.external.ShowData",
|
||||||
|
"properties" : {
|
||||||
|
"showNumber" : "5"
|
||||||
|
}
|
||||||
|
} ],
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b4262a"
|
||||||
|
}
|
||||||
|
} ],
|
||||||
|
"name" : "g1",
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b22615"
|
||||||
|
}
|
||||||
|
} ],
|
||||||
|
"conditions" : [ {
|
||||||
|
"entry" : "f4",
|
||||||
|
"after" : "g1"
|
||||||
|
}, {
|
||||||
|
"entry" : "f3",
|
||||||
|
"after" : "g1"
|
||||||
|
} ],
|
||||||
|
"uuid" : "8a80d63f720cdd2301723b7745b22614"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,80 @@
|
||||||
|
package cn.piflow.api
|
||||||
|
|
||||||
|
import org.apache.http.client.config.RequestConfig
|
||||||
|
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
|
||||||
|
import org.apache.http.entity.StringEntity
|
||||||
|
import org.apache.http.impl.client.HttpClientBuilder
|
||||||
|
import org.apache.http.util.EntityUtils
|
||||||
|
|
||||||
|
object HTTPClientStartMockDataFlow {
|
||||||
|
|
||||||
|
def main(args: Array[String]): Unit = {
|
||||||
|
|
||||||
|
val json =
|
||||||
|
"""
|
||||||
|
|{
|
||||||
|
| "flow": {
|
||||||
|
| "name": "MockData",
|
||||||
|
| "executorMemory": "1g",
|
||||||
|
| "executorNumber": "1",
|
||||||
|
| "uuid": "8a80d63f720cdd2301723b7461d92600",
|
||||||
|
| "paths": [
|
||||||
|
| {
|
||||||
|
| "inport": "",
|
||||||
|
| "from": "MockData",
|
||||||
|
| "to": "ShowData",
|
||||||
|
| "outport": ""
|
||||||
|
| }
|
||||||
|
| ],
|
||||||
|
| "executorCores": "1",
|
||||||
|
| "driverMemory": "1g",
|
||||||
|
| "stops": [
|
||||||
|
| {
|
||||||
|
| "name": "MockData",
|
||||||
|
| "bundle": "cn.piflow.bundle.common.MockData",
|
||||||
|
| "uuid": "8a80d63f720cdd2301723b7461d92604",
|
||||||
|
| "properties": {
|
||||||
|
| "schema": "title:String, author:String, age:Int",
|
||||||
|
| "count": "10"
|
||||||
|
| },
|
||||||
|
| "customizedProperties": {
|
||||||
|
|
|
||||||
|
| }
|
||||||
|
| },
|
||||||
|
| {
|
||||||
|
| "name": "ShowData",
|
||||||
|
| "bundle": "cn.piflow.bundle.external.ShowData",
|
||||||
|
| "uuid": "8a80d63f720cdd2301723b7461d92602",
|
||||||
|
| "properties": {
|
||||||
|
| "showNumber": "5"
|
||||||
|
| },
|
||||||
|
| "customizedProperties": {
|
||||||
|
|
|
||||||
|
| }
|
||||||
|
| }
|
||||||
|
| ]
|
||||||
|
| }
|
||||||
|
|}
|
||||||
|
""".stripMargin
|
||||||
|
|
||||||
|
val url = "http://10.0.85.83:8001/flow/start"
|
||||||
|
val timeout = 1800
|
||||||
|
val requestConfig = RequestConfig.custom()
|
||||||
|
.setConnectTimeout(timeout*1000)
|
||||||
|
.setConnectionRequestTimeout(timeout*1000)
|
||||||
|
.setSocketTimeout(timeout*1000).build()
|
||||||
|
|
||||||
|
val client = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build()
|
||||||
|
|
||||||
|
val post:HttpPost = new HttpPost(url)
|
||||||
|
post.addHeader("Content-Type", "application/json")
|
||||||
|
post.setEntity(new StringEntity(json))
|
||||||
|
|
||||||
|
|
||||||
|
val response:CloseableHttpResponse = client.execute(post)
|
||||||
|
val entity = response.getEntity
|
||||||
|
val str = EntityUtils.toString(entity,"UTF-8")
|
||||||
|
println("Code is " + str)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
51
readMe.txt
51
readMe.txt
|
|
@ -5,51 +5,16 @@ mvn install:install-file -Dfile=/opt/project/piflow/piflow-bundle/lib/java_memca
|
||||||
mvn install:install-file -Dfile=/opt/project/piflow/piflow-bundle/lib/ojdbc6-11.2.0.3.jar -DgroupId=oracle -DartifactId=ojdbc6 -Dversion=11.2.0.3 -Dpackaging=jar
|
mvn install:install-file -Dfile=/opt/project/piflow/piflow-bundle/lib/ojdbc6-11.2.0.3.jar -DgroupId=oracle -DartifactId=ojdbc6 -Dversion=11.2.0.3 -Dpackaging=jar
|
||||||
mvn install:install-file -Dfile=/opt/project/piflow/piflow-bundle/lib/edtftpj.jar -DgroupId=ftpClient -DartifactId=edtftp -Dversion=1.0.0 -Dpackaging=jar
|
mvn install:install-file -Dfile=/opt/project/piflow/piflow-bundle/lib/edtftpj.jar -DgroupId=ftpClient -DartifactId=edtftp -Dversion=1.0.0 -Dpackaging=jar
|
||||||
|
|
||||||
2.packaging
|
2.Packaging by Intellij
|
||||||
|
1)Edit Configurations --> add Maven
|
||||||
|
Command line: clean package -Dmaven.test.skip=true -X
|
||||||
|
2)Build piflow-server-0.9.jar
|
||||||
|
|
||||||
clean package -Dmaven.test.skip=true -U
|
3.run main class in Intellij
|
||||||
|
|
||||||
3.set SPARK_HOME in Configurations
|
|
||||||
Edit Configurations --> Application(HttpService) --> Configurations --> Environment Variable
|
|
||||||
|
|
||||||
4. yarn log aggregation
|
|
||||||
Edit yarn-site.xml, add the following content
|
|
||||||
<property>
|
|
||||||
<name>yarn.log-aggregation-enable</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property>
|
|
||||||
<name>yarn.nodemanager.log-aggregation.debug-enabled</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property>
|
|
||||||
<name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
|
|
||||||
<value>3600</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
5.kafka related jars are needed to put on the spark cluster
|
|
||||||
spark-streaming-kafka-0-10_2.11-2.1.0.jar
|
|
||||||
kafka_2.11-2.1.1.jar
|
|
||||||
kafka-clients-2.1.1.jar
|
|
||||||
|
|
||||||
start kafka server: ./bin/kafka-server-start.sh -daemon config/server.properties
|
|
||||||
stop kafka server: ./bin/kafka-server-stop.sh
|
|
||||||
start kafka producer: ./bin/kafka-console-producer.sh --broker-list master:9092,slave1:9092,slave2:9092 --topic streaming
|
|
||||||
start kafka consumer: ./bin/kafka-console-consumer.sh --zookeeper master:2181,slave1:2181,slave2:2181 --topic streaming
|
|
||||||
list topics:
|
|
||||||
./bin/kafka-topics.sh --list --zookeeper master:2181,slave1:2181,slave2:2181
|
|
||||||
./bin/kafka-topics.sh --list --zookeeper master:2181,slave1:2181,slave2:2181
|
|
||||||
create topics:
|
|
||||||
./bin/kafka-topics.sh --create --zookeeper master:2181,slave1:2181,slave2:2181 --replication-factor 3 --partictions 3 --topic newTopic
|
|
||||||
|
|
||||||
|
|
||||||
6.flume related jars are needed to put on the spark cluster
|
1)Edit Configurations --> Application
|
||||||
spark-streaming-flume_2.11-2.1.0.jar
|
Main class: cn.piflow.api.Main
|
||||||
|
Environment Variable: SPARK_HOME=/opt/spark-2.2.0-bin-hadoop2.6;
|
||||||
|
|
||||||
start flume agent: bin/flume-ng agent -n streamingAgent -c conf -f conf/streaming.conf -Dflume.root.logger=INFO,console
|
|
||||||
|
|
||||||
7.socket text stream
|
|
||||||
nc -lk 9999
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue