opensci
/
piflow
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

YangQiDong

This commit is contained in:
yanfqidong0604 2019-01-18 16:27:46 +08:00
parent d027f5e589
commit 6d483b7461
11 changed files with 447 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

389
piflow-bundle/piflow-bundle.iml Normal file
View File

@ -0,0 +1,389 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/main/scala" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test/scala" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: org.biojava:biojava-structure:4.0.0" level="project" />
<orderEntry type="library" name="Maven: org.biojava:biojava-alignment:4.0.0" level="project" />
<orderEntry type="library" name="Maven: org.biojava:biojava-phylo:4.0.0" level="project" />
<orderEntry type="library" name="Maven: org.biojava.thirdparty:forester:1.005" level="project" />
<orderEntry type="library" name="Maven: org.biojava:biojava-core:4.0.0" level="project" />
<orderEntry type="library" name="Maven: java3d:vecmath:1.3.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.logging.log4j:log4j-slf4j-impl:2.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.logging.log4j:log4j-core:2.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hive:hive-jdbc:1.2.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hive:hive-common:1.2.1" level="project" />
<orderEntry type="library" name="Maven: log4j:apache-log4j-extras:1.2.17" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
<orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.ant:ant:1.9.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.ant:ant-launcher:1.9.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hive:hive-service:1.2.1" level="project" />
<orderEntry type="library" name="Maven: net.sf.jpam:jpam:1.1" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty.aggregate:jetty-all:7.6.0.v20120127" level="project" />
<orderEntry type="library" name="Maven: javax.servlet:servlet-api:2.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.geronimo.specs:geronimo-jta_1.1_spec:1.1.1" level="project" />
<orderEntry type="library" name="Maven: javax.mail:mail:1.4.1" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.geronimo.specs:geronimo-jaspic_1.0_spec:1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.geronimo.specs:geronimo-annotation_1.0_spec:1.1.1" level="project" />
<orderEntry type="library" name="Maven: asm:asm-commons:3.1" level="project" />
<orderEntry type="library" name="Maven: asm:asm-tree:3.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hive:hive-serde:1.2.1" level="project" />
<orderEntry type="library" name="Maven: net.sf.opencsv:opencsv:2.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.hive:hive-metastore:1.2.1" level="project" />
<orderEntry type="library" name="Maven: com.jolbox:bonecp:0.8.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.apache.derby:derby:10.10.2.0" level="project" />
<orderEntry type="library" name="Maven: org.datanucleus:datanucleus-api-jdo:3.2.6" level="project" />
<orderEntry type="library" name="Maven: org.datanucleus:datanucleus-rdbms:3.2.9" level="project" />
<orderEntry type="library" name="Maven: commons-pool:commons-pool:1.5.4" level="project" />
<orderEntry type="library" name="Maven: javax.jdo:jdo-api:3.0.1" level="project" />
<orderEntry type="library" name="Maven: javax.transaction:jta:1.1" level="project" />
<orderEntry type="library" name="Maven: org.antlr:antlr-runtime:3.4" level="project" />
<orderEntry type="library" name="Maven: org.antlr:stringtemplate:3.2.1" level="project" />
<orderEntry type="library" name="Maven: antlr:antlr:2.7.7" level="project" />
<orderEntry type="library" name="Maven: org.apache.hive:hive-shims:1.2.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hive.shims:hive-shims-common:1.2.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.hive.shims:hive-shims-0.20S:1.2.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.hive.shims:hive-shims-0.23:1.2.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.hadoop:hadoop-yarn-server-resourcemanager:2.6.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.jersey:jersey-json:1.9" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.xml.bind:jaxb-impl:2.2.3-1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.codehaus.jackson:jackson-xc:1.8.3" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.jersey.contribs:jersey-guice:1.9" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.codehaus.jettison:jettison:1.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.jersey:jersey-client:1.9" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice:2.6.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.hive.shims:hive-shims-scheduler:1.2.1" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.thrift:libthrift:0.9.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" />
<orderEntry type="library" name="Maven: jline:jline:0.9.94" level="project" />
<orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.curator:curator-client:2.6.0" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.5" level="project" />
<orderEntry type="library" name="Maven: org.mongodb:mongodb-driver:3.4.1" level="project" />
<orderEntry type="library" name="Maven: org.mongodb:mongodb-driver-core:3.4.1" level="project" />
<orderEntry type="library" name="Maven: org.mongodb:bson:3.4.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:7.2.0" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpmime:4.5.3" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.4" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.woodstox:woodstox-core-asl:4.4.1" level="project" />
<orderEntry type="library" name="Maven: org.noggit:noggit:0.8" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.7.7" level="project" />
<orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
<orderEntry type="library" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
<orderEntry type="module" module-name="piflow-core" />
<orderEntry type="library" name="Maven: org.quartz-scheduler:quartz:2.3.0" level="project" />
<orderEntry type="library" name="Maven: com.mchange:c3p0:0.9.5.2" level="project" />
<orderEntry type="library" name="Maven: com.mchange:mchange-commons-java:0.2.11" level="project" />
<orderEntry type="library" name="Maven: com.zaxxer:HikariCP-java6:2.3.13" level="project" />
<orderEntry type="library" name="Maven: org.clapper:classutil_2.11:1.3.0" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm:6.1.1" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm-commons:6.1.1" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm-tree:6.1.1" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm-analysis:6.1.1" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm-util:6.1.1" level="project" />
<orderEntry type="library" name="Maven: org.clapper:grizzled-scala_2.11:4.4.2" level="project" />
<orderEntry type="library" name="Maven: org.reflections:reflections:0.9.9" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:15.0" level="project" />
<orderEntry type="library" name="Maven: org.javassist:javassist:3.18.2-GA" level="project" />
<orderEntry type="library" name="Maven: com.google.code.findbugs:annotations:2.0.1" level="project" />
<orderEntry type="library" name="Maven: com.chuusai:shapeless_2.11:2.3.1" level="project" />
<orderEntry type="library" name="Maven: org.typelevel:macro-compat_2.11:1.1.1" level="project" />
<orderEntry type="library" name="Maven: com.sksamuel.scrimage:scrimage-core_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-core:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-jpeg:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-metadata:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.common:common-lang:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.common:common-io:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.common:common-image:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.drewnoakes:metadata-extractor:2.8.1" level="project" />
<orderEntry type="library" name="Maven: com.adobe.xmp:xmpcore:5.1.2" level="project" />
<orderEntry type="library" name="Maven: ar.com.hjg:pngj:2.1.0" level="project" />
<orderEntry type="library" name="Maven: com.sksamuel.scrimage:scrimage-io-extra_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-bmp:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-icns:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-iff:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-pcx:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-pict:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-pdf:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-pnm:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-psd:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-sgi:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-tiff:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-tga:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.twelvemonkeys.imageio:imageio-thumbsdb:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.sksamuel.scrimage:scrimage-filters_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.25" level="project" />
<orderEntry type="library" name="Maven: net.liftweb:lift-json_2.11:2.6.1" level="project" />
<orderEntry type="library" name="Maven: org.scala-lang:scalap:2.11.1" level="project" />
<orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.4.1" level="project" />
<orderEntry type="library" name="Maven: com.databricks:spark-xml_2.11:0.4.1" level="project" />
<orderEntry type="library" name="Maven: redis.clients:jedis:2.9.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.kafka:kafka-clients:0.11.0.0" level="project" />
<orderEntry type="library" name="Maven: net.jpountz.lz4:lz4:1.3.0" level="project" />
<orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.1.2.6" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch-hadoop:6.4.2" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch:5.6.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-core:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-analyzers-common:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-backward-codecs:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-grouping:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-highlighter:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-join:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-memory:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-misc:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-queries:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-queryparser:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-sandbox:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-spatial:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-spatial-extras:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-spatial3d:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-suggest:6.6.1" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch:securesm:1.1" level="project" />
<orderEntry type="library" name="Maven: net.sf.jopt-simple:jopt-simple:5.0.2" level="project" />
<orderEntry type="library" name="Maven: com.carrotsearch:hppc:0.7.1" level="project" />
<orderEntry type="library" name="Maven: joda-time:joda-time:2.9.5" level="project" />
<orderEntry type="library" name="Maven: org.yaml:snakeyaml:1.15" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.8.6" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.8.6" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.6" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.6" level="project" />
<orderEntry type="library" name="Maven: com.tdunning:t-digest:3.0" level="project" />
<orderEntry type="library" name="Maven: org.hdrhistogram:HdrHistogram:2.1.9" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch:jna:4.4.0-1" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-mllib_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-streaming_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-graphx_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: com.github.fommil.netlib:core:1.1.2" level="project" />
<orderEntry type="library" name="Maven: net.sourceforge.f2j:arpack_combined_all:0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-mllib-local_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.scalanlp:breeze_2.11:0.12" level="project" />
<orderEntry type="library" name="Maven: org.scalanlp:breeze-macros_2.11:0.12" level="project" />
<orderEntry type="library" name="Maven: com.github.rwl:jtransforms:2.4.0" level="project" />
<orderEntry type="library" name="Maven: org.spire-math:spire_2.11:0.7.4" level="project" />
<orderEntry type="library" name="Maven: org.spire-math:spire-macros_2.11:0.7.4" level="project" />
<orderEntry type="library" name="Maven: org.jpmml:pmml-model:1.2.15" level="project" />
<orderEntry type="library" name="Maven: org.jpmml:pmml-schema:1.2.15" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-tags_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.scalatest:scalatest_2.11:2.2.6" level="project" />
<orderEntry type="library" name="Maven: org.spark-project.spark:unused:1.0.0" level="project" />
<orderEntry type="library" name="Maven: org.jsoup:jsoup:1.10.3" level="project" />
<orderEntry type="library" name="Maven: org.json:json:20151123" level="project" />
<orderEntry type="library" name="Maven: org.biojava:core:1.9.1" level="project" />
<orderEntry type="library" name="Maven: commons-dbcp:commons-dbcp:1.2.2" level="project" />
<orderEntry type="library" name="Maven: org.biojava:bytecode:1.9.1" level="project" />
<orderEntry type="library" name="Maven: com.memcached:java_memcached-release:2.6.6" level="project" />
<orderEntry type="library" name="Maven: jdbc_oracle:ojdbc:6.0.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.flume:flume-ng-core:1.8.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.flume:flume-ng-sdk:1.8.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.flume:flume-ng-configuration:1.8.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.flume:flume-ng-auth:1.8.0" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.8" level="project" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
<orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.avro:avro-ipc:1.7.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.velocity:velocity:1.7" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty:3.9.4.Final" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:servlet-api:2.5-20110124" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.26" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jetty:6.1.26" level="project" />
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.2.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.mina:mina-core:2.0.4" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-all:4.0.42.Final" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-client:1.2.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-annotations:1.2.6" level="project" />
<orderEntry type="library" name="Maven: com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-common:1.2.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-protocol:1.2.6" level="project" />
<orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.htrace:htrace-core:3.1.0-incubating" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.9.13" level="project" />
<orderEntry type="library" name="Maven: org.jruby.jcodings:jcodings:1.0.8" level="project" />
<orderEntry type="library" name="Maven: org.jruby.joni:joni:2.1.2" level="project" />
<orderEntry type="library" name="Maven: com.yammer.metrics:metrics-core:2.2.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-common:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-annotations:2.5.1" level="project" />
<orderEntry type="module-library">
<library name="Maven: jdk.tools:jdk.tools:1.6">
<CLASSES>
<root url="jar:///opt/java/jdk1.8.0_181/lib/tools.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="library" name="Maven: xmlenc:xmlenc:0.52" level="project" />
<orderEntry type="library" name="Maven: commons-el:commons-el:1.0" level="project" />
<orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />
<orderEntry type="library" name="Maven: commons-digester:commons-digester:1.8" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />
<orderEntry type="library" name="Maven: com.jcraft:jsch:0.1.42" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-server:1.2.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-procedure:1.2.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-common:tests:1.2.6" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.hbase:hbase-prefix-tree:1.2.6" level="project" />
<orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-hadoop-compat:1.2.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.hbase:hbase-hadoop2-compat:1.2.6" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-core:1.9" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-server:1.9" level="project" />
<orderEntry type="library" name="Maven: asm:asm:3.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math:2.2" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-sslengine:6.1.26" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jsp-2.1:6.1.14" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jsp-api-2.1:6.1.14" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:servlet-api-2.5:6.1.14" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.9.13" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.9.13" level="project" />
<orderEntry type="library" name="Maven: tomcat:jasper-compiler:5.5.23" level="project" />
<orderEntry type="library" name="Maven: tomcat:jasper-runtime:5.5.23" level="project" />
<orderEntry type="library" name="Maven: org.jamon:jamon-runtime:2.4.1" level="project" />
<orderEntry type="library" name="Maven: com.lmax:disruptor:3.3.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-client:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-hdfs:2.5.1" level="project" />
<orderEntry type="library" name="Maven: commons-daemon:commons-daemon:1.0.13" level="project" />
<orderEntry type="library" name="Maven: net.sourceforge.jexcelapi:jxl:2.6.12" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:3.11-beta1" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi:3.11-beta1" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-schemas:3.11-beta1" level="project" />
<orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:2.3.0" level="project" />
<orderEntry type="library" name="Maven: net.sf.json-lib:json-lib:jdk15:2.4" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.8.0" level="project" />
<orderEntry type="library" name="Maven: net.sf.ezmorph:ezmorph:1.0.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-pool2:2.4.2" level="project" />
<orderEntry type="library" name="Maven: org.scala-lang:scala-library:2.11.8" level="project" />
<orderEntry type="library" name="Maven: org.scala-lang:scala-reflect:2.11.8" level="project" />
<orderEntry type="library" name="Maven: org.scala-lang:scala-compiler:2.11.8" level="project" />
<orderEntry type="library" name="Maven: org.scala-lang.modules:scala-xml_2.11:1.0.4" level="project" />
<orderEntry type="library" name="Maven: org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.11" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-core_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.avro:avro-mapred:hadoop2:1.7.7" level="project" />
<orderEntry type="library" name="Maven: org.apache.avro:avro-ipc:tests:1.7.7" level="project" />
<orderEntry type="library" name="Maven: com.twitter:chill_2.11:0.8.0" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:kryo-shaded:3.0.3" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:minlog:1.3.0" level="project" />
<orderEntry type="library" name="Maven: org.objenesis:objenesis:2.1" level="project" />
<orderEntry type="library" name="Maven: com.twitter:chill-java:0.8.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.xbean:xbean-asm5-shaded:4.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-launcher_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-network-common_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.6.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-network-shuffle_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-unsafe_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: net.java.dev.jets3t:jets3t:0.7.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.curator:curator-recipes:2.4.0" level="project" />
<orderEntry type="library" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.5" level="project" />
<orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:jul-to-slf4j:1.7.16" level="project" />
<orderEntry type="library" name="Maven: com.ning:compress-lzf:1.0.3" level="project" />
<orderEntry type="library" name="Maven: org.roaringbitmap:RoaringBitmap:0.5.11" level="project" />
<orderEntry type="library" name="Maven: commons-net:commons-net:2.2" level="project" />
<orderEntry type="library" name="Maven: org.json4s:json4s-jackson_2.11:3.2.11" level="project" />
<orderEntry type="library" name="Maven: org.json4s:json4s-core_2.11:3.2.11" level="project" />
<orderEntry type="library" name="Maven: org.json4s:json4s-ast_2.11:3.2.11" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.jersey.core:jersey-client:2.22.2" level="project" />
<orderEntry type="library" name="Maven: javax.ws.rs:javax.ws.rs-api:2.0.1" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.hk2:hk2-api:2.4.0-b34" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.hk2:hk2-utils:2.4.0-b34" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.hk2.external:aopalliance-repackaged:2.4.0-b34" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.hk2.external:javax.inject:2.4.0-b34" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.hk2:hk2-locator:2.4.0-b34" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.jersey.core:jersey-common:2.22.2" level="project" />
<orderEntry type="library" name="Maven: javax.annotation:javax.annotation-api:1.2" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.jersey.bundles.repackaged:jersey-guava:2.22.2" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.hk2:osgi-resource-locator:1.0.1" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.jersey.core:jersey-server:2.22.2" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.jersey.media:jersey-media-jaxb:2.22.2" level="project" />
<orderEntry type="library" name="Maven: javax.validation:validation-api:1.1.0.Final" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.jersey.containers:jersey-container-servlet:2.22.2" level="project" />
<orderEntry type="library" name="Maven: org.glassfish.jersey.containers:jersey-container-servlet-core:2.22.2" level="project" />
<orderEntry type="library" name="Maven: com.clearspring.analytics:stream:2.7.0" level="project" />
<orderEntry type="library" name="Maven: io.dropwizard.metrics:metrics-core:3.1.2" level="project" />
<orderEntry type="library" name="Maven: io.dropwizard.metrics:metrics-jvm:3.1.2" level="project" />
<orderEntry type="library" name="Maven: io.dropwizard.metrics:metrics-json:3.1.2" level="project" />
<orderEntry type="library" name="Maven: io.dropwizard.metrics:metrics-graphite:3.1.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.6.5" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.module:jackson-module-scala_2.11:2.6.5" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.module:jackson-module-paranamer:2.6.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.ivy:ivy:2.4.0" level="project" />
<orderEntry type="library" name="Maven: oro:oro:2.0.8" level="project" />
<orderEntry type="library" name="Maven: net.razorvine:pyrolite:4.13" level="project" />
<orderEntry type="library" name="Maven: net.sf.py4j:py4j:0.10.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-crypto:1.0.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-sql_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: com.univocity:univocity-parsers:2.2.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-sketch_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-catalyst_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.janino:janino:3.0.0" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.janino:commons-compiler:3.0.0" level="project" />
<orderEntry type="library" name="Maven: org.antlr:antlr4-runtime:4.5.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.parquet:parquet-column:1.8.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.parquet:parquet-common:1.8.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.parquet:parquet-encoding:1.8.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.parquet:parquet-hadoop:1.8.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.parquet:parquet-format:2.3.0-incubating" level="project" />
<orderEntry type="library" name="Maven: org.apache.parquet:parquet-jackson:1.8.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-hive_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: com.twitter:parquet-hadoop-bundle:1.6.0" level="project" />
<orderEntry type="library" name="Maven: org.spark-project.hive:hive-exec:1.2.1.spark2" level="project" />
<orderEntry type="library" name="Maven: javolution:javolution:5.5.1" level="project" />
<orderEntry type="library" name="Maven: org.antlr:ST4:4.0.4" level="project" />
<orderEntry type="library" name="Maven: com.googlecode.javaewah:JavaEWAH:0.3.2" level="project" />
<orderEntry type="library" name="Maven: org.iq80.snappy:snappy:0.2" level="project" />
<orderEntry type="library" name="Maven: stax:stax-api:1.0.1" level="project" />
<orderEntry type="library" name="Maven: org.spark-project.hive:hive-metastore:1.2.1.spark2" level="project" />
<orderEntry type="library" name="Maven: org.apache.calcite:calcite-avatica:1.2.0-incubating" level="project" />
<orderEntry type="library" name="Maven: org.apache.calcite:calcite-core:1.2.0-incubating" level="project" />
<orderEntry type="library" name="Maven: org.apache.calcite:calcite-linq4j:1.2.0-incubating" level="project" />
<orderEntry type="library" name="Maven: net.hydromatic:eigenbase-properties:1.1.5" level="project" />
<orderEntry type="library" name="Maven: org.jodd:jodd-core:3.5.2" level="project" />
<orderEntry type="library" name="Maven: org.datanucleus:datanucleus-core:3.2.10" level="project" />
<orderEntry type="library" name="Maven: org.apache.thrift:libfb303:0.9.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.spark:spark-yarn_2.11:2.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.2.0" level="project" />
<orderEntry type="library" name="Maven: com.google.inject.extensions:guice-servlet:3.0" level="project" />
<orderEntry type="library" name="Maven: com.google.inject:guice:3.0" level="project" />
<orderEntry type="library" name="Maven: javax.inject:javax.inject:1" level="project" />
<orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.2.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-server-web-proxy:2.2.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.2.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.2.0" level="project" />
<orderEntry type="library" name="Maven: com.h2database:h2:1.4.197" level="project" />
</component>
</module>

5
piflow-bundle/src/main/scala/cn/piflow/bundle/memcache/ComplementByMemcache.scala
View File

@ -10,7 +10,6 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
class ComplementByMemcache extends ConfigurableStop {
override val authorEmail: String = "yangqidong@cnic.cn"
@ -76,10 +75,6 @@ class ComplementByMemcache extends ConfigurableStop {
val schema: StructType = StructType(fields)
val df: DataFrame = session.createDataFrame(rowRDD,schema)
//println("############################################################")
//df.show()
//println("############################################################")
out.write(df)
}

4
piflow-bundle/src/main/scala/cn/piflow/bundle/memcache/GetMemcache.scala
View File

@ -74,10 +74,6 @@ class GetMemcache extends ConfigurableStop{
val s: StructType = StructType(fields)
val df: DataFrame = session.createDataFrame(rowRDD,s)
//println("############################################################")
//df.show()
//println("############################################################")
out.write(df)
}

95
piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/EmblParser.scala
View File

@ -22,7 +22,6 @@ class EmblParser extends ConfigurableStop{
override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
val session = pec.get[SparkSession]()
val inDf: DataFrame = in.read()
@ -51,73 +50,31 @@ class EmblParser extends ConfigurableStop{
var bis: BufferedInputStream =null
// var df: DataFrame =null
// var d: DataFrame =null
// var jsonRDD: RDD[String] =null
inDf.collect().foreach(row => {
var n : Int =0
pathStr = row.get(0).asInstanceOf[String]
println("#############################################")
println("start parser ^^^" + pathStr)
println("#############################################")
var fdis: FSDataInputStream = fs.open(new Path(pathStr))
// if(pathStr.equals("hdfs://10.0.88.70:9000/yqd/weishengwu/refseq/bacteria.1.genomic.gbff")) {
var br: BufferedReader = new BufferedReader(new InputStreamReader(fdis))
var sequences: RichSequenceIterator = CustomIOTools.IOTools.readEMBLDNA (br, null)
var fdis: FSDataInputStream = fs.open(new Path(pathStr))
// var fdis: FSDataInputStream = fs.open(new Path("hdfs://10.0.88.70:9000/yqd/weishengwu/refseq/bacteria.1.1.genomic.fna.gz"))
while (sequences.hasNext) {
n += 1
var seq: RichSequence = sequences.nextRichSequence()
var doc: JSONObject = new JSONObject
Process.processEMBL_EnsemblSeq(seq, doc)
jsonStr = doc.toString
println("start " + n)
// var gzipout: GZIPInputStream = new GZIPInputStream(fdis)
// var br: BufferedReader = new BufferedReader(new InputStreamReader(gzipout))
var br: BufferedReader = new BufferedReader(new InputStreamReader(fdis))
var sequences: RichSequenceIterator = CustomIOTools.IOTools.readEMBLDNA (br, null)
while (sequences.hasNext) {
n += 1
var seq: RichSequence = sequences.nextRichSequence()
var doc: JSONObject = new JSONObject
Process.processEMBL_EnsemblSeq(seq, doc)
jsonStr = doc.toString
println("start " + n)
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
} else {
bis = new BufferedInputStream(new ByteArrayInputStream(("," + jsonStr).getBytes()))
}
val buff: Array[Byte] = new Array[Byte](1048576)
var count: Int = bis.read(buff)
while (count != -1) {
fdos.write(buff, 0, count)
fdos.flush()
count = bis.read(buff)
}
/* if(n==1){
jsonRDD = session.sparkContext.makeRDD(jsonStr :: Nil)
df = session.read.json(jsonRDD)
}else{
jsonRDD = session.sparkContext.makeRDD(jsonStr :: Nil)
d = session.read.json(jsonRDD)
df = df.union(d.toDF(df.columns:_*))
}*/
fdos.flush()
bis = null
seq = null
doc = null
// jsonRDD = null
// d = null
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
} else {
bis = new BufferedInputStream(new ByteArrayInputStream(("," + jsonStr).getBytes()))
}
bis = new BufferedInputStream(new ByteArrayInputStream(("]").getBytes()))
val buff: Array[Byte] = new Array[Byte](1048576)
var count: Int = bis.read(buff)
@ -126,22 +83,30 @@ class EmblParser extends ConfigurableStop{
fdos.flush()
count = bis.read(buff)
}
fdos.flush()
// }
bis = null
seq = null
doc = null
}
bis = new BufferedInputStream(new ByteArrayInputStream(("]").getBytes()))
val buff: Array[Byte] = new Array[Byte](1048576)
var count: Int = bis.read(buff)
while (count != -1) {
fdos.write(buff, 0, count)
fdos.flush()
count = bis.read(buff)
}
fdos.flush()
})
fdos.close()
println("start parser HDFSjsonFile")
val df: DataFrame = session.read.json(hdfsPathTemporary)
//println("############################################################")
// println(df.count())
//df.show(20)
//println("############################################################")
out.write(df)
}
override def setProperties(map: Map[String, Any]): Unit = {

10
piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/Ensembl_gff3Parser.scala
View File

@ -90,12 +90,12 @@ class Ensembl_gff3Parser extends ConfigurableStop{
br = new BufferedReader(new InputStreamReader(fdis))
var eachStr:String=null
while((eachStr = br.readLine()) != null && eachStr != null /*&& n<=10000*/ ){
while((eachStr = br.readLine()) != null && eachStr != null){
doc = parser.parserGff3(eachStr)
jsonStr = doc.toString
if(jsonStr.length > 2){
n +=1
println("start " + n + " String" /*+ jsonStr*/)
println("start " + n + " String")
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
} else {
@ -133,14 +133,8 @@ class Ensembl_gff3Parser extends ConfigurableStop{
bis.close()
fdos.close()
// println("start parser HDFSjsonFile --------------------")
val df: DataFrame = session.read.json(hdfsPathTemporary)
//println("############################################################")
// println(df.count())
// df.show(20)
// df.printSchema()
//println("############################################################")
out.write(df)
}
}

10
piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/GeneParser.scala
View File

@ -59,8 +59,6 @@ class GeneParser extends ConfigurableStop{
pathStr = row.get(0).asInstanceOf[String]
println("########################## start parser ^^^" + pathStr)
var fdis: FSDataInputStream = fs.open(new Path(pathStr))
var br: BufferedReader = new BufferedReader(new InputStreamReader(fdis))
@ -68,7 +66,7 @@ class GeneParser extends ConfigurableStop{
var line:String=""
var doc:JSONObject=null
while ((line=br.readLine()) != null /*&& n<1000*/){
while ((line=br.readLine()) != null){
if( ! line.startsWith("#")){
n += 1
doc=new JSONObject()
@ -87,7 +85,6 @@ class GeneParser extends ConfigurableStop{
}
}
jsonStr = doc.toString
println(n+"^^^^^^^^^^^^^^^^^^^^")
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
} else {
@ -122,13 +119,8 @@ println(n+"^^^^^^^^^^^^^^^^^^^^")
fdos.close()
println("start parser HDFSjsonFile")
val df: DataFrame = session.read.json(hdfsPathTemporary)
//println("############################################################")
//println(df.count())
//df.show(20)
//println("############################################################")
out.write(df)

7
piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/MicrobeGenomeDataParser.scala
View File

@ -63,7 +63,6 @@ class MicrobeGenomeDataParser extends ConfigurableStop{
inDf.collect().foreach(row => {
pathStr = row.get(0).asInstanceOf[String]
println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! start parser ^^^" + pathStr)
fdis = fs.open(new Path(pathStr))
br = new BufferedReader(new InputStreamReader(fdis))
sequences = CustomIOTools.IOTools.readGenbankProtein(br, null)
@ -77,7 +76,6 @@ class MicrobeGenomeDataParser extends ConfigurableStop{
Process.processSingleSequence(seq,doc)
jsonStr = doc.toString
println("start " + n + "String\\\n" /*+ jsonStr*/)
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
@ -119,11 +117,6 @@ class MicrobeGenomeDataParser extends ConfigurableStop{
println("start parser HDFSjsonFile")
val df: DataFrame = session.read.json(hdfsPathTemporary)
//println("############################################################")
// println(df.count())
//df.show(20)
// df.printSchema()
//println("############################################################")
out.write(df)
}

10
piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/PDBParser.scala
View File

@ -64,14 +64,12 @@ class PDBParser extends ConfigurableStop{
var n:Int=0
inDf.collect().foreach(row => {
pathStr = row.get(0).asInstanceOf[String]
println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! start parser ^^^" + pathStr)
pdb = new PDB(pathStr,fs)
doc = pdb.getDoc
jsonStr = doc.toString
n +=1
println("start " + n + "String\\\n" /*+ jsonStr*/)
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
@ -108,17 +106,9 @@ class PDBParser extends ConfigurableStop{
bis.close()
fdos.close()
// println("start parser HDFSjsonFile --------------------")
val df: DataFrame = session.read.json(hdfsPathTemporary)
// println("############################################################")
// println(df.count())
// df.show(20)
// df.printSchema()
// println("############################################################")
out.write(df)
}

8
piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/PfamDataParser.scala
View File

@ -63,7 +63,6 @@ class PfamDataParser extends ConfigurableStop{
var n:Int=0
inDf.collect().foreach(row => {
pathStr = row.get(0).asInstanceOf[String]
println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! start parser ^^^" + pathStr)
fdis = fs.open(new Path(pathStr))
br = new BufferedReader(new InputStreamReader(fdis))
@ -74,7 +73,6 @@ class PfamDataParser extends ConfigurableStop{
hasAnotherSequence = Pfam.process(br,doc)
jsonStr = doc.toString
println("start " + n + "String\\\n" /*+ jsonStr*/)
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
@ -112,14 +110,8 @@ class PfamDataParser extends ConfigurableStop{
bis.close()
fdos.close()
// println("start parser HDFSjsonFile --------------------")
val df: DataFrame = session.read.json(hdfsPathTemporary)
// println("############################################################")
// println(df.count())
// df.show(20)
// df.printSchema()
// println("############################################################")
out.write(df)
}

56
piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/RefseqParser.scala
View File

@ -57,11 +57,6 @@ class RefseqParser extends ConfigurableStop{
var n : Int =0
pathStr = row.get(0).asInstanceOf[String]
println("#############################################")
println("start parser ^^^" + pathStr)
println("#############################################")
// if(pathStr.equals("hdfs://10.0.88.70:9000/yqd/weishengwu/refseq/bacteria.1.genomic.gbff")) {
var fdis: FSDataInputStream = fs.open(new Path(pathStr))
@ -69,33 +64,32 @@ class RefseqParser extends ConfigurableStop{
var sequences: RichSequenceIterator = CustomIOTools.IOTools.readGenbankProtein(br, null)
while (sequences.hasNext) {
n += 1
var seq: RichSequence = sequences.nextRichSequence()
var doc: JSONObject = new JSONObject
Process.processSingleSequence(seq, doc)
jsonStr = doc.toString
println("start " + n)
while (sequences.hasNext) {
n += 1
var seq: RichSequence = sequences.nextRichSequence()
var doc: JSONObject = new JSONObject
Process.processSingleSequence(seq, doc)
jsonStr = doc.toString
println("start " + n)
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
} else {
bis = new BufferedInputStream(new ByteArrayInputStream(("," + jsonStr).getBytes()))
}
var count: Int = bis.read(buff)
while (count != -1) {
fdos.write(buff, 0, count)
fdos.flush()
count = bis.read(buff)
}
fdos.flush()
bis = null
seq = null
doc = null
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
} else {
bis = new BufferedInputStream(new ByteArrayInputStream(("," + jsonStr).getBytes()))
}
// }
var count: Int = bis.read(buff)
while (count != -1) {
fdos.write(buff, 0, count)
fdos.flush()
count = bis.read(buff)
}
fdos.flush()
bis = null
seq = null
doc = null
}
})
bis = new BufferedInputStream(new ByteArrayInputStream(("]").getBytes()))
@ -112,10 +106,6 @@ class RefseqParser extends ConfigurableStop{
println("start parser HDFSjsonFile")
val df: DataFrame = session.read.json(hdfsPathTemporary)
// println("############################################################")
// println(df.count())
// df.show(20)
// println("############################################################")
out.write(df)

11
piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/Swissprot_TrEMBLDataParser.scala
View File

@ -83,12 +83,11 @@ class Swissprot_TrEMBLDataParser extends ConfigurableStop{
var n:Int=0
inDf.collect().foreach(row => {
pathStr = row.get(0).asInstanceOf[String]
println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! start parser ^^^" + pathStr)
fdis = fs.open(new Path(pathStr))
br = new BufferedReader(new InputStreamReader(fdis))
sequences = CustomIOTools.IOTools.readUniProt(br,null)
while (sequences.hasNext /* && n<10000*/) {
while (sequences.hasNext) {
n += 1
@ -96,7 +95,7 @@ class Swissprot_TrEMBLDataParser extends ConfigurableStop{
seq = sequences.nextRichSequence()
Process.processUniprotSeq(seq,doc)
jsonStr = doc.toString
println("start " + n + "String\\\n" /*+ jsonStr*/)
println("start " + n + "String\\\n" )
if (n == 1) {
bis = new BufferedInputStream(new ByteArrayInputStream(("[" + jsonStr).getBytes()))
@ -133,14 +132,8 @@ class Swissprot_TrEMBLDataParser extends ConfigurableStop{
bis.close()
fdos.close()
// println("start parser HDFSjsonFile --------------------")
val df: DataFrame = session.read.json(hdfsPathTemporary)
// println("############################################################")
// println(df.count())
// df.show(20)
// df.printSchema()
// println("############################################################")
out.write(df)
}