opensci
/
piflow
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

genbank001

This commit is contained in:
yanggang 2018-11-01 17:33:40 +08:00
parent 06bdb859d2
commit 1d623d70b7
8 changed files with 14 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
piflow-bundle/src/main/scala/cn/piflow/bundle/genBankParse/LoadFromFtpUrl.scala → piflow-bundle/src/main/scala/cn/piflow/bundle/ftp/LoadFromFtpUrl.scala
View File

@ -1,19 +1,16 @@
package cn.piflow.bundle.genBankParse package cn.piflow.bundle.ftp
import java.io._ import java.io._
import java.net.URL import java.net.URL
import java.util
import java.util.ArrayList import java.util.ArrayList
import cn.piflow.conf.bean.PropertyDescriptor import cn.piflow.conf.bean.PropertyDescriptor
import cn.piflow.conf.util.{ImageUtil, MapUtil} import cn.piflow.conf.util.{ImageUtil, MapUtil}
import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroupEnum} import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroupEnum}
import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext} import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.SparkSession
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.jsoup.select.Elements import org.jsoup.select.Elements
import shapeless.ops.hlist.Unzip
import sun.net.ftp.FtpProtocolException import sun.net.ftp.FtpProtocolException
@ -162,7 +159,7 @@ class LoadFromFtpUrl extends ConfigurableStop{
} }
override def getGroup(): List[String] = { override def getGroup(): List[String] = {
List(StopGroupEnum.GenBankParseGroup.toString) List(StopGroupEnum.FtpGroup.toString)
} }
def initialize(ctx: ProcessContext): Unit = { def initialize(ctx: ProcessContext): Unit = {

11
piflow-bundle/src/main/scala/cn/piflow/bundle/genBankParse/UnGz.scala → piflow-bundle/src/main/scala/cn/piflow/bundle/ftp/UnGz.scala
View File

@ -1,8 +1,4 @@
package cn.piflow.bundle.genBankParse package cn.piflow.bundle.ftp
import java.io._
import java.net.URL
import java.util.ArrayList
import cn.piflow.bundle.util.UnGzUtil import cn.piflow.bundle.util.UnGzUtil
import cn.piflow.conf.bean.PropertyDescriptor import cn.piflow.conf.bean.PropertyDescriptor
@ -10,9 +6,6 @@ import cn.piflow.conf.util.{ImageUtil, MapUtil}
import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroupEnum} import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroupEnum}
import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext} import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.{Row, SparkSession}
import org.jsoup.Jsoup
import org.jsoup.select.Elements
import sun.net.ftp.FtpProtocolException
class UnGz extends ConfigurableStop{ class UnGz extends ConfigurableStop{
@ -80,7 +73,7 @@ class UnGz extends ConfigurableStop{
} }
override def getGroup(): List[String] = { override def getGroup(): List[String] = {
List(StopGroupEnum.GenBankParseGroup.toString) List(StopGroupEnum.FtpGroup.toString)
} }
def initialize(ctx: ProcessContext): Unit = { def initialize(ctx: ProcessContext): Unit = {

8
piflow-bundle/src/main/scala/cn/piflow/bundle/genBankParse/GenBankParse.scala → piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/GenBankParse.scala
View File

@ -1,4 +1,4 @@
package cn.piflow.bundle.genBankParse package cn.piflow.bundle.microorganism
import java.io._ import java.io._
import java.net.URL import java.net.URL
@ -7,8 +7,8 @@ import java.util
import java.util.ArrayList import java.util.ArrayList
import java.util.zip.GZIPInputStream import java.util.zip.GZIPInputStream
import cn.piflow.bundle.genBankParse.util.CustomIOTools import cn.piflow.bundle.microorganism.util.CustomIOTools
import cn.piflow.bundle.genBankParse.util.Process import cn.piflow.bundle.microorganism.util.Process
import cn.piflow.conf.bean.PropertyDescriptor import cn.piflow.conf.bean.PropertyDescriptor
import cn.piflow.conf.util.{ImageUtil, MapUtil} import cn.piflow.conf.util.{ImageUtil, MapUtil}
import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroupEnum} import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroupEnum}
@ -204,7 +204,7 @@ class GenBankParse extends ConfigurableStop{
} }
override def getGroup(): List[String] = { override def getGroup(): List[String] = {
List(StopGroupEnum.GenBankParseGroup.toString) List(StopGroupEnum.MicroorganismGroup.toString)
} }
def initialize(ctx: ProcessContext): Unit = { def initialize(ctx: ProcessContext): Unit = {

2
piflow-bundle/src/main/scala/cn/piflow/bundle/genBankParse/util/CustomGenbankFormat.java → piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomGenbankFormat.java
View File

@ -1,4 +1,4 @@
package cn.piflow.bundle.genBankParse.util; package cn.piflow.bundle.microorganism.util;

2
piflow-bundle/src/main/scala/cn/piflow/bundle/genBankParse/util/CustomIOTools.java → piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomIOTools.java
View File

@ -1,4 +1,4 @@
package cn.piflow.bundle.genBankParse.util; package cn.piflow.bundle.microorganism.util;
import org.biojava.bio.BioError; import org.biojava.bio.BioError;

2
piflow-bundle/src/main/scala/cn/piflow/bundle/genBankParse/util/Process.java → piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/Process.java
View File

@ -1,4 +1,4 @@
package cn.piflow.bundle.genBankParse.util; package cn.piflow.bundle.microorganism.util;

2
piflow-bundle/src/main/scala/cn/piflow/conf/StopGroup.scala
View File

@ -18,7 +18,7 @@ case object KafkaGroup extends StopGroup
case object ESGroup extends StopGroup case object ESGroup extends StopGroup
case object UrlGroup extends StopGroup case object UrlGroup extends StopGroup
case object HdfsGroup extends StopGroup case object HdfsGroup extends StopGroup
case object GenBankParseGroup extends StopGroup case object MicroorganismGroup extends StopGroup

2
piflow-bundle/src/main/scala/cn/piflow/conf/StopGroupEnum.scala
View File

@ -22,6 +22,6 @@ object StopGroupEnum extends Enumeration {
val RDFGroup = Value("RDFGroup") val RDFGroup = Value("RDFGroup")
val UrlGroup= Value("UrlGroup") val UrlGroup= Value("UrlGroup")
val HdfsGroup= Value("HdfsGroup") val HdfsGroup= Value("HdfsGroup")
val GenBankParseGroup= Value("GenBankParseGroup") val MicroorganismGroup= Value("MicroorganismGroup")
} }