From 3d8dd3720ce075533332bdf9fb87c971b3404e7d Mon Sep 17 00:00:00 2001 From: yanggang Date: Tue, 9 Apr 2019 19:08:54 +0800 Subject: [PATCH] mv all *.java to java package --- .../microorganism/util/CustomEMBLFormat.java | 0 .../util/CustomEnsemblFormat.java | 0 .../util/CustomGenbankFormat.java | 0 .../microorganism/util/CustomIOTools.java | 0 .../util/CustomUniProtFormat.java | 0 .../microorganism/util/KeggPathway.java | 0 .../piflow/bundle/microorganism/util/PDB.java | 8 ++++--- .../bundle/microorganism/util/Pfam.java | 9 ++++++-- .../bundle/microorganism/util/Process.java | 0 .../bundle/microorganism/util/ProcessNew.java | 0 .../microorganism/util/ReadSection.java | 0 .../cn/piflow/bundle/util/ExcelToJson.java | 4 ---- .../bundle/util/JedisClusterImplSer.java | 2 +- .../scala/cn/piflow/bundle/hdfs/PutHdfs.scala | 22 ++++++++++--------- 14 files changed, 25 insertions(+), 20 deletions(-) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/CustomEMBLFormat.java (100%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/CustomEnsemblFormat.java (100%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/CustomGenbankFormat.java (100%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/CustomIOTools.java (100%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/CustomUniProtFormat.java (100%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/KeggPathway.java (100%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/PDB.java (98%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/Pfam.java (99%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/Process.java (100%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/ProcessNew.java (100%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/microorganism/util/ReadSection.java (100%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/util/ExcelToJson.java (97%) rename piflow-bundle/src/main/{scala => java}/cn/piflow/bundle/util/JedisClusterImplSer.java (97%) diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomEMBLFormat.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomEMBLFormat.java similarity index 100% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomEMBLFormat.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomEMBLFormat.java diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomEnsemblFormat.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomEnsemblFormat.java similarity index 100% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomEnsemblFormat.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomEnsemblFormat.java diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomGenbankFormat.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomGenbankFormat.java similarity index 100% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomGenbankFormat.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomGenbankFormat.java diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomIOTools.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomIOTools.java similarity index 100% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomIOTools.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomIOTools.java diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomUniProtFormat.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomUniProtFormat.java similarity index 100% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/CustomUniProtFormat.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/CustomUniProtFormat.java diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/KeggPathway.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/KeggPathway.java similarity index 100% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/KeggPathway.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/KeggPathway.java diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/PDB.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/PDB.java similarity index 98% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/PDB.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/PDB.java index 5bc54ca..965e372 100644 --- a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/PDB.java +++ b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/PDB.java @@ -1,6 +1,7 @@ package cn.piflow.bundle.microorganism.util; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.log4j.Logger; import org.biojava.nbio.structure.*; @@ -8,13 +9,14 @@ import org.biojava.nbio.structure.io.PDBFileReader; import org.json.JSONArray; import org.json.JSONObject; -import java.io.*; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; -import java.util.zip.GZIPInputStream; -import org.apache.hadoop.fs.FileSystem; public class PDB { diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/Pfam.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/Pfam.java similarity index 99% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/Pfam.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/Pfam.java index 9315841..5e3271a 100644 --- a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/Pfam.java +++ b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/Pfam.java @@ -4,8 +4,13 @@ package cn.piflow.bundle.microorganism.util; import org.biojava.bio.seq.io.ParseException; import org.json.JSONArray; import org.json.JSONObject; -import java.io.*; -import java.util.*; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/Process.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/Process.java similarity index 100% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/Process.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/Process.java diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/ProcessNew.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/ProcessNew.java similarity index 100% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/ProcessNew.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/ProcessNew.java diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/ReadSection.java b/piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/ReadSection.java similarity index 100% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/microorganism/util/ReadSection.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/microorganism/util/ReadSection.java diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/util/ExcelToJson.java b/piflow-bundle/src/main/java/cn/piflow/bundle/util/ExcelToJson.java similarity index 97% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/util/ExcelToJson.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/util/ExcelToJson.java index 44b7781..5e0de8d 100644 --- a/piflow-bundle/src/main/scala/cn/piflow/bundle/util/ExcelToJson.java +++ b/piflow-bundle/src/main/java/cn/piflow/bundle/util/ExcelToJson.java @@ -7,7 +7,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.poi.hssf.usermodel.HSSFDateUtil; import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; @@ -15,9 +14,6 @@ import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/util/JedisClusterImplSer.java b/piflow-bundle/src/main/java/cn/piflow/bundle/util/JedisClusterImplSer.java similarity index 97% rename from piflow-bundle/src/main/scala/cn/piflow/bundle/util/JedisClusterImplSer.java rename to piflow-bundle/src/main/java/cn/piflow/bundle/util/JedisClusterImplSer.java index 8c9206b..3fbdbb6 100644 --- a/piflow-bundle/src/main/scala/cn/piflow/bundle/util/JedisClusterImplSer.java +++ b/piflow-bundle/src/main/java/cn/piflow/bundle/util/JedisClusterImplSer.java @@ -8,7 +8,7 @@ import java.io.IOException; import java.io.ObjectStreamException; import java.io.Serializable; -public class JedisClusterImplSer implements Serializable { +public class JedisClusterImplSer implements Serializable { private static final long serialVersionUID = -51L; diff --git a/piflow-bundle/src/main/scala/cn/piflow/bundle/hdfs/PutHdfs.scala b/piflow-bundle/src/main/scala/cn/piflow/bundle/hdfs/PutHdfs.scala index 92ad841..5517005 100644 --- a/piflow-bundle/src/main/scala/cn/piflow/bundle/hdfs/PutHdfs.scala +++ b/piflow-bundle/src/main/scala/cn/piflow/bundle/hdfs/PutHdfs.scala @@ -29,14 +29,14 @@ class PutHdfs extends ConfigurableStop{ config.set("fs.defaultFS",hdfsUrl) val fs = FileSystem.get(config) - if (types=="json"){ - inDF.repartition(partition).write.json(hdfsUrl+hdfsPath) - } else if (types=="csv"){ - inDF.repartition(partition).write.csv(hdfsUrl+hdfsPath) - } else { - //parquet - inDF.repartition(partition).write.save(hdfsUrl+hdfsPath) - } + if (types=="json"){ + inDF.repartition(partition).write.json(hdfsUrl+hdfsPath) + } else if (types=="csv"){ + inDF.repartition(partition).write.csv(hdfsUrl+hdfsPath) + } else { + //parquet + inDF.repartition(partition).write.save(hdfsUrl+hdfsPath) + } } override def setProperties(map: Map[String, Any]): Unit = { @@ -50,8 +50,10 @@ class PutHdfs extends ConfigurableStop{ var descriptor : List[PropertyDescriptor] = List() val hdfsPath = new PropertyDescriptor().name("hdfsPath").displayName("hdfsPath").defaultValue("").required(true) val hdfsUrl = new PropertyDescriptor().name("hdfsUrl").displayName("hdfsUrl").defaultValue("").required(true) - val types = new PropertyDescriptor().name("types").displayName("json,csv,parquet").defaultValue("").required(true) - val partition = new PropertyDescriptor().name("partition").displayName("repartition").defaultValue("").required(true) + val types = new PropertyDescriptor().name("types").displayName("json,csv,parquet").description("json,csv,parquet") + .defaultValue("csv").allowableValues(Set("json","csv","parquet")).required(true) + + val partition = new PropertyDescriptor().name("partition").displayName("repartition").description("partition").defaultValue("").required(true) descriptor = partition :: descriptor descriptor = types :: descriptor descriptor = hdfsPath :: descriptor