From 4e2f35a1e626a9621c277bba080f5392034f3836 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Wed, 7 Jun 2017 19:22:55 +0800 Subject: [PATCH 01/15] update constants --- .../tsfile/spark/common/FilterOperator.java | 3 - .../edu/thu/tsfile/spark/common/Operator.java | 2 - .../thu/tsfile/spark/common/SQLConstant.java | 143 ++++++++++++++++++ .../spark/optimizer/DNFFilterOptimizer.java | 4 +- .../spark/optimizer/PhysicalOptimizer.java | 4 +- .../spark/optimizer/RemoveNotOptimizer.java | 6 +- .../cn/edu/thu/tsfile/spark/Converter.scala | 5 +- .../edu/thu/tsfile/spark/DefaultSource.scala | 8 +- .../edu/thu/tsfile/spark/QueryPlanTest.java | 2 +- src/test/resources/test.tsfile | Bin 0 -> 1233 bytes src/test/resources/tsfile/test1.tsfile | Bin 1231 -> 1233 bytes src/test/resources/tsfile/test2.tsfile | Bin 497 -> 499 bytes .../edu/thu/tsfile/spark/ConverterTest.scala | 2 +- .../cn/edu/thu/tsfile/spark/TSFileSuit.scala | 2 +- 14 files changed, 160 insertions(+), 21 deletions(-) create mode 100644 src/main/java/cn/edu/thu/tsfile/spark/common/SQLConstant.java create mode 100644 src/test/resources/test.tsfile diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/FilterOperator.java b/src/main/java/cn/edu/thu/tsfile/spark/common/FilterOperator.java index dbf8760..907d7fe 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/FilterOperator.java +++ b/src/main/java/cn/edu/thu/tsfile/spark/common/FilterOperator.java @@ -1,8 +1,5 @@ package cn.edu.thu.tsfile.spark.common; - - -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant; import cn.edu.thu.tsfile.timeseries.utils.StringContainer; import java.util.ArrayList; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/Operator.java b/src/main/java/cn/edu/thu/tsfile/spark/common/Operator.java index 38580f3..3661719 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/Operator.java +++ b/src/main/java/cn/edu/thu/tsfile/spark/common/Operator.java @@ -1,7 +1,5 @@ package cn.edu.thu.tsfile.spark.common; -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant; - /** * This class is a superclass of all operator. * @author kangrong diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/SQLConstant.java b/src/main/java/cn/edu/thu/tsfile/spark/common/SQLConstant.java new file mode 100644 index 0000000..a857db2 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfile/spark/common/SQLConstant.java @@ -0,0 +1,143 @@ +package cn.edu.thu.tsfile.spark.common; + +import cn.edu.thu.tsfile.timeseries.read.qp.Path; + +import java.util.HashMap; +import java.util.Map; + +/** + * this class contains several constants used in SQL. + * + * @author kangrong + * + */ +public class SQLConstant { + public static final String MREGE_EXTENSION = "merge"; + public static final String ERR_EXTENSION = "err"; + public static final String PATH_SEPARATOR = "."; + public static final String PATH_SEPARATER_NO_REGEX = "\\."; + + public static final String DEFAULT_DELTA_OBJECT_TYPE = "defalut_delta_object_type"; + + public static final String RESERVED_TIME = "time"; + public static final String RESERVED_FREQ = "freq"; + public static final String RESERVED_DELTA_OBJECT = "delta_object"; + public static final String IS_AGGREGATION = "IS_AGGREGATION"; + + public static final String lineFeedSignal = "\n"; + public static final String ROOT = "root"; + public static final String METADATA_PARAM_EQUAL = "="; + + public static final int KW_AND = 1; + public static final int KW_OR = 2; + public static final int KW_NOT = 3; + + public static final int EQUAL = 11; + public static final int NOTEQUAL = 12; + public static final int LESSTHANOREQUALTO = 13; + public static final int LESSTHAN = 14; + public static final int GREATERTHANOREQUALTO = 15; + public static final int GREATERTHAN = 16; + public static final int EQUAL_NS = 17; + + public static final int TOK_SELECT = 21; + public static final int TOK_FROM = 22; + public static final int TOK_WHERE = 23; + public static final int TOK_INSERT = 24; + public static final int TOK_DELETE = 25; + public static final int TOK_UPDATE = 26; + public static final int TOK_QUERY = 27; + // public static final int TOK_VIRTUAL_TABLE = 30; + // public static final int TOK_TABNAME = 31; + // public static final int TOK_TABREF = 32; + + public static final int TOK_AUTHOR_CREATE = 41; + public static final int TOK_AUTHOR_DROP = 42; + public static final int TOK_AUTHOR_GRANT = 43; + public static final int TOK_AUTHOR_REVOKE = 44; + + public static final int TOK_DATALOAD = 45; + + public static final int TOK_METADATA_CREATE = 51; + public static final int TOK_METADATA_DELETE = 52; + public static final int TOK_METADATA_SET_FILE_LEVEL = 53; + public static final int TOK_PORPERTY_CREATE = 54; + public static final int TOK_PORPERTY_ADD_LABEL = 55; + public static final int TOK_PORPERTY_DELETE_LABEL = 56; + public static final int TOK_PORPERTY_LINK = 57; + public static final int TOK_PORPERTY_UNLINK = 58; + + + + public static Map tokenSymbol = new HashMap(); + public static Map tokenNames = new HashMap(); + public static Map reverseWords = new HashMap(); + + static { + tokenSymbol.put(KW_AND, "&"); + tokenSymbol.put(KW_OR, "|"); + tokenSymbol.put(KW_NOT, "!"); + tokenSymbol.put(EQUAL, "="); + tokenSymbol.put(NOTEQUAL, "<>"); + tokenSymbol.put(EQUAL_NS, "<=>"); + tokenSymbol.put(LESSTHANOREQUALTO, "<="); + tokenSymbol.put(LESSTHAN, "<"); + tokenSymbol.put(GREATERTHANOREQUALTO, ">="); + tokenSymbol.put(GREATERTHAN, ">"); + } + + static { + tokenNames.put(KW_AND, "and"); + tokenNames.put(KW_OR, "or"); + tokenNames.put(KW_NOT, "not"); + tokenNames.put(EQUAL, "equal"); + tokenNames.put(NOTEQUAL, "not_equal"); + tokenNames.put(EQUAL_NS, "equal_ns"); + tokenNames.put(LESSTHANOREQUALTO, "lessthan_or_equalto"); + tokenNames.put(LESSTHAN, "lessthan"); + tokenNames.put(GREATERTHANOREQUALTO, "greaterthan_or_equalto"); + tokenNames.put(GREATERTHAN, "greaterthan"); + + tokenNames.put(TOK_SELECT, "TOK_SELECT"); + tokenNames.put(TOK_FROM, "TOK_FROM"); + tokenNames.put(TOK_WHERE, "TOK_WHERE"); + tokenNames.put(TOK_INSERT, "TOK_INSERT"); + tokenNames.put(TOK_DELETE, "TOK_DELETE"); + tokenNames.put(TOK_UPDATE, "TOK_UPDATE"); + tokenNames.put(TOK_QUERY, "TOK_QUERY"); + + tokenNames.put(TOK_AUTHOR_CREATE, "TOK_AUTHOR_CREATE"); + tokenNames.put(TOK_AUTHOR_DROP, "TOK_AUTHOR_DROP"); + tokenNames.put(TOK_AUTHOR_GRANT, "TOK_AUTHOR_GRANT"); + tokenNames.put(TOK_AUTHOR_REVOKE, "TOK_AUTHOR_REVOKE"); + tokenNames.put(TOK_DATALOAD, "TOK_DATALOAD"); + + tokenNames.put(TOK_METADATA_CREATE, "TOK_METADATA_CREATE"); + tokenNames.put(TOK_METADATA_DELETE, "TOK_METADATA_DELETE"); + tokenNames.put(TOK_METADATA_SET_FILE_LEVEL, "TOK_METADATA_SET_FILE_LEVEL"); + tokenNames.put(TOK_PORPERTY_CREATE, "TOK_PORPERTY_CREATE"); + tokenNames.put(TOK_PORPERTY_ADD_LABEL, "TOK_PORPERTY_ADD_LABEL"); + tokenNames.put(TOK_PORPERTY_DELETE_LABEL, "TOK_PORPERTY_DELETE_LABEL"); + tokenNames.put(TOK_PORPERTY_LINK, "TOK_PORPERTY_LINK"); + tokenNames.put(TOK_PORPERTY_UNLINK, "TOK_PORPERTY_UNLINK"); + + } + + static { + reverseWords.put(KW_AND, KW_OR); + reverseWords.put(KW_OR, KW_AND); + reverseWords.put(EQUAL, NOTEQUAL); + reverseWords.put(NOTEQUAL, EQUAL); + reverseWords.put(LESSTHAN, GREATERTHANOREQUALTO); + reverseWords.put(GREATERTHANOREQUALTO, LESSTHAN); + reverseWords.put(LESSTHANOREQUALTO, GREATERTHAN); + reverseWords.put(GREATERTHAN, LESSTHANOREQUALTO); + } + + public static boolean isReservedPath(Path pathStr) { + return pathStr.equals(SQLConstant.RESERVED_TIME) + || pathStr.equals(SQLConstant.RESERVED_FREQ) + || pathStr.equals(SQLConstant.RESERVED_DELTA_OBJECT); + + } +} diff --git a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/DNFFilterOptimizer.java b/src/main/java/cn/edu/thu/tsfile/spark/optimizer/DNFFilterOptimizer.java index 2654eb8..d5bdb01 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/DNFFilterOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/spark/optimizer/DNFFilterOptimizer.java @@ -7,8 +7,8 @@ import java.util.ArrayList; import java.util.List; -import static cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant.KW_AND; -import static cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant.KW_OR; +import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_AND; +import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_OR; public class DNFFilterOptimizer implements IFilterOptimizer { diff --git a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java b/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java index 94a5b85..def007d 100644 --- a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java @@ -27,7 +27,7 @@ public List optimize(SingleQuery singleQuery, List paths, T QueryEngine queryEngine = new QueryEngine(in); List actualDeltaObjects = queryEngine.getAllDeltaObjectUIDByPartition(start, end); - List actualSeries = queryEngine.getAllSeries(); + List actualSeries = queryEngine.getAllSeriesSchema(); List validDeltaObjects = new ArrayList<>(); @@ -76,7 +76,7 @@ public List optimize(SingleQuery singleQuery, List paths, T validDeltaObjects.addAll(queryEngine.getAllDeltaObjectUIDByPartition(start, end)); } - List fileSeries = queryEngine.getAllSeries(); + List fileSeries = queryEngine.getAllSeriesSchema(); Set seriesSet = new HashSet<>(); for(SeriesSchema series: fileSeries) { seriesSet.add(series.name); diff --git a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/RemoveNotOptimizer.java b/src/main/java/cn/edu/thu/tsfile/spark/optimizer/RemoveNotOptimizer.java index 4319184..2dc270a 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/RemoveNotOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/spark/optimizer/RemoveNotOptimizer.java @@ -3,15 +3,17 @@ import cn.edu.thu.tsfile.spark.common.BasicOperator; import cn.edu.thu.tsfile.spark.common.FilterOperator; +import cn.edu.thu.tsfile.spark.common.SQLConstant; import cn.edu.thu.tsfile.spark.exception.BasicOperatorException; import cn.edu.thu.tsfile.spark.exception.RemoveNotException; -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; -import static cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant.*; +import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_AND; +import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_NOT; +import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_OR; public class RemoveNotOptimizer implements IFilterOptimizer { diff --git a/src/main/scala/cn/edu/thu/tsfile/spark/Converter.scala b/src/main/scala/cn/edu/thu/tsfile/spark/Converter.scala index 771d3a1..0f8c3ee 100755 --- a/src/main/scala/cn/edu/thu/tsfile/spark/Converter.scala +++ b/src/main/scala/cn/edu/thu/tsfile/spark/Converter.scala @@ -5,9 +5,8 @@ import java.util import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader import cn.edu.thu.tsfile.file.metadata.enums.TSDataType import cn.edu.thu.tsfile.hadoop.io.HDFSInputStream -import cn.edu.thu.tsfile.spark.common.{BasicOperator, FilterOperator, TSQueryPlan} +import cn.edu.thu.tsfile.spark.common.{BasicOperator, FilterOperator, SQLConstant, TSQueryPlan} import cn.edu.thu.tsfile.timeseries.read.metadata.SeriesSchema -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant import cn.edu.thu.tsfile.timeseries.read.query.{QueryConfig, QueryEngine} import cn.edu.thu.tsfile.timeseries.read.readSupport.Field import org.apache.hadoop.conf.Configuration @@ -48,7 +47,7 @@ object Converter { files.foreach(f => { val in = new HDFSInputStream(f.getPath, conf) val queryEngine = new QueryEngine(in) - val series = queryEngine.getAllSeries + val series = queryEngine.getAllSeriesSchema series.foreach(s => { if(!seriesSet.contains(s.name)) { seriesSet += s.name diff --git a/src/main/scala/cn/edu/thu/tsfile/spark/DefaultSource.scala b/src/main/scala/cn/edu/thu/tsfile/spark/DefaultSource.scala index ad57405..a06e714 100755 --- a/src/main/scala/cn/edu/thu/tsfile/spark/DefaultSource.scala +++ b/src/main/scala/cn/edu/thu/tsfile/spark/DefaultSource.scala @@ -7,7 +7,7 @@ import java.util import cn.edu.thu.tsfile.common.constant.QueryConstant import cn.edu.thu.tsfile.hadoop.io.HDFSInputStream import cn.edu.thu.tsfile.spark.DefaultSource.SerializableConfiguration -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant +import cn.edu.thu.tsfile.spark.common.SQLConstant import cn.edu.thu.tsfile.timeseries.read.query.QueryDataSet import cn.edu.thu.tsfile.timeseries.read.readSupport.{Field, RowRecord} import com.esotericsoftware.kryo.io.{Input, Output} @@ -56,8 +56,8 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { //check if the path is given options.getOrElse(DefaultSource.path, throw new TSFileDataSourceException(s"${DefaultSource.path} must be specified for tsfile DataSource")) - //get union series in tsfile - var tsfileSchema = Converter.getUnionSeries(files, conf) + //get union series in TsFile + val tsfileSchema = Converter.getUnionSeries(files, conf) Converter.toSparkSqlSchema(tsfileSchema) } @@ -143,7 +143,7 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { while (priorityQueue.nonEmpty && !hasNext) { //get a record from priorityQueue - var tmpRecord = priorityQueue.dequeue() + val tmpRecord = priorityQueue.dequeue() //insert a record to priorityQueue if (dataSets(tmpRecord.index).hasNextRecord) { priorityQueue.enqueue(Record(dataSets(tmpRecord.index).getNextRecord, tmpRecord.index)) diff --git a/src/test/java/cn/edu/thu/tsfile/spark/QueryPlanTest.java b/src/test/java/cn/edu/thu/tsfile/spark/QueryPlanTest.java index 0f46bd5..470d650 100644 --- a/src/test/java/cn/edu/thu/tsfile/spark/QueryPlanTest.java +++ b/src/test/java/cn/edu/thu/tsfile/spark/QueryPlanTest.java @@ -3,10 +3,10 @@ import cn.edu.thu.tsfile.spark.common.BasicOperator; import cn.edu.thu.tsfile.spark.common.FilterOperator; +import cn.edu.thu.tsfile.spark.common.SQLConstant; import cn.edu.thu.tsfile.spark.common.TSQueryPlan; import cn.edu.thu.tsfile.spark.exception.QueryProcessorException; import cn.edu.thu.tsfile.timeseries.read.LocalFileInput; -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/resources/test.tsfile b/src/test/resources/test.tsfile new file mode 100644 index 0000000000000000000000000000000000000000..f733bfc64107f6d498140881eaf3c800bb9732cf GIT binary patch literal 1233 zcmai!KTE?v7{)J`v%SW_BHWQe2N#Qil>QaPp`}7CMZ`fIT*N;u2!h%K(M`lb`~+G$ zs`vq1+#DSgiiqImAc%{D_yK$`O^!>07@hLMB2;hua&PI~l;kLptePO}DJVr}q-v>Z~r^XzA-hjJ8Qj z@9v&3S0k#E&7Y#!4Q>@yk!}Q*s41QUVZB9(PL>ZGhZ?I%2P#})ESlad7v|(|rw>)L zX_CrAK95qAOX14cmx1_s8UkYcyYoYXP{Jp(;M0B&ptiU0rr diff --git a/src/test/scala/cn/edu/thu/tsfile/spark/ConverterTest.scala b/src/test/scala/cn/edu/thu/tsfile/spark/ConverterTest.scala index 66b307f..84f407e 100644 --- a/src/test/scala/cn/edu/thu/tsfile/spark/ConverterTest.scala +++ b/src/test/scala/cn/edu/thu/tsfile/spark/ConverterTest.scala @@ -4,9 +4,9 @@ import java.io.File import java.util import cn.edu.thu.tsfile.file.metadata.enums.{TSDataType, TSEncoding} +import cn.edu.thu.tsfile.spark.common.SQLConstant import cn.edu.thu.tsfile.timeseries.read.LocalFileInput import cn.edu.thu.tsfile.timeseries.read.metadata.SeriesSchema -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant import cn.edu.thu.tsfile.timeseries.read.query.QueryConfig import cn.edu.thu.tsfile.timeseries.read.readSupport.Field import org.apache.spark.sql.sources.{Filter, GreaterThan, LessThan, Or} diff --git a/src/test/scala/cn/edu/thu/tsfile/spark/TSFileSuit.scala b/src/test/scala/cn/edu/thu/tsfile/spark/TSFileSuit.scala index d21dcbf..de43cf4 100644 --- a/src/test/scala/cn/edu/thu/tsfile/spark/TSFileSuit.scala +++ b/src/test/scala/cn/edu/thu/tsfile/spark/TSFileSuit.scala @@ -2,7 +2,7 @@ package cn.edu.thu.tsfile.spark import java.io.File -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant +import cn.edu.thu.tsfile.spark.common.SQLConstant import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types._ import org.junit.Assert From c0f7ec212067a1968b320f3ac9fcd1b0503ab2c1 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Fri, 9 Jun 2017 16:46:28 +0800 Subject: [PATCH 02/15] update constant with tsfile --- .../java/cn/edu/thu/tsfile/spark/QueryProcessor.java | 2 +- .../cn/edu/thu/tsfile/spark/common/BasicOperator.java | 1 - .../cn/edu/thu/tsfile/spark/common/SparkConstant.java | 9 --------- .../thu/tsfile/spark/optimizer/PhysicalOptimizer.java | 6 +----- 4 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 src/main/java/cn/edu/thu/tsfile/spark/common/SparkConstant.java diff --git a/src/main/java/cn/edu/thu/tsfile/spark/QueryProcessor.java b/src/main/java/cn/edu/thu/tsfile/spark/QueryProcessor.java index 46ec8b3..5e298f6 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/QueryProcessor.java +++ b/src/main/java/cn/edu/thu/tsfile/spark/QueryProcessor.java @@ -2,6 +2,7 @@ import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; import cn.edu.thu.tsfile.spark.common.FilterOperator; +import cn.edu.thu.tsfile.spark.common.SQLConstant; import cn.edu.thu.tsfile.spark.common.SingleQuery; import cn.edu.thu.tsfile.spark.common.TSQueryPlan; import cn.edu.thu.tsfile.spark.exception.QueryOperatorException; @@ -10,7 +11,6 @@ import cn.edu.thu.tsfile.spark.optimizer.MergeSingleFilterOptimizer; import cn.edu.thu.tsfile.spark.optimizer.PhysicalOptimizer; import cn.edu.thu.tsfile.spark.optimizer.RemoveNotOptimizer; -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant; import java.io.IOException; import java.util.ArrayList; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/BasicOperator.java b/src/main/java/cn/edu/thu/tsfile/spark/common/BasicOperator.java index 14476cf..676cec1 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/BasicOperator.java +++ b/src/main/java/cn/edu/thu/tsfile/spark/common/BasicOperator.java @@ -2,7 +2,6 @@ import cn.edu.thu.tsfile.spark.exception.BasicOperatorException; -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant; /** * basic operators include < > >= <= !=. diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/SparkConstant.java b/src/main/java/cn/edu/thu/tsfile/spark/common/SparkConstant.java deleted file mode 100644 index 2020b90..0000000 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/SparkConstant.java +++ /dev/null @@ -1,9 +0,0 @@ -package cn.edu.thu.tsfile.spark.common; - -/** - * Created by qiaojialin on 2017/4/24. - */ -public class SparkConstant { - public static final String DELTA_OBJECT_SEPARATOR = "+"; - public static final String DELTA_OBJECT_VALUE_SEPARATOR = ":"; -} diff --git a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java b/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java index def007d..fffc958 100644 --- a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java @@ -1,12 +1,8 @@ package cn.edu.thu.tsfile.spark.optimizer; import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; -import cn.edu.thu.tsfile.spark.common.BasicOperator; -import cn.edu.thu.tsfile.spark.common.FilterOperator; -import cn.edu.thu.tsfile.spark.common.SingleQuery; -import cn.edu.thu.tsfile.spark.common.TSQueryPlan; +import cn.edu.thu.tsfile.spark.common.*; import cn.edu.thu.tsfile.timeseries.read.metadata.SeriesSchema; -import cn.edu.thu.tsfile.timeseries.read.qp.SQLConstant; import cn.edu.thu.tsfile.timeseries.read.query.QueryEngine; import java.io.IOException; From 3e30b78d80878cbf987e759819024a691fcda039 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Fri, 9 Jun 2017 16:48:10 +0800 Subject: [PATCH 03/15] update artifactid in pom --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 694e245..eb473aa 100644 --- a/pom.xml +++ b/pom.xml @@ -3,8 +3,8 @@ 4.0.0 cn.edu.thu - tsfile - 1.0-SNAPSHOT + tsfile-spark-connector + 0.1.0 From 3ac950736edc0acaa731cb958fc3a5ea6f4cbb95 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Fri, 9 Jun 2017 18:40:04 +0800 Subject: [PATCH 04/15] change package --- README.md | 8 ++++++- .../edu/thu/tsfile/{spark => }/Executor.java | 2 +- .../tsfile/{spark => }/QueryProcessor.java | 22 +++++++++---------- .../{spark => }/common/BasicOperator.java | 4 ++-- .../{spark => }/common/FilterOperator.java | 2 +- .../tsfile/{spark => }/common/Operator.java | 2 +- .../{spark => }/common/SQLConstant.java | 2 +- .../{spark => }/common/SingleQuery.java | 2 +- .../{spark => }/common/TSQueryPlan.java | 2 +- .../exception/BasicOperatorException.java | 2 +- .../exception/DNFOptimizeException.java | 2 +- .../exception/LogicalOptimizeException.java | 2 +- .../exception/MergeFilterException.java | 2 +- .../exception/QueryOperatorException.java | 4 +--- .../exception/QueryProcessorException.java | 2 +- .../exception/RemoveNotException.java | 2 +- .../optimizer/DNFFilterOptimizer.java | 10 ++++----- .../tsfile/optimizer/IFilterOptimizer.java | 16 ++++++++++++++ .../optimizer/MergeSingleFilterOptimizer.java | 6 ++--- .../optimizer/PhysicalOptimizer.java | 4 ++-- .../optimizer/RemoveNotOptimizer.java | 18 +++++++-------- .../spark/optimizer/IFilterOptimizer.java | 16 -------------- .../thu/tsfile/{spark => }/Converter.scala | 4 ++-- .../tsfile/{spark => }/DefaultSource.scala | 6 ++--- .../edu/thu/tsfile/{spark => }/package.scala | 6 ++--- .../thu/tsfile/{spark => }/CreateTSFile.java | 2 +- .../thu/tsfile/{spark => }/QueryPlanTest.java | 12 +++++----- .../tsfile/{spark => }/ConverterTest.scala | 4 ++-- .../thu/tsfile/{spark => }/TSFileSuit.scala | 10 ++++----- 29 files changed, 90 insertions(+), 86 deletions(-) rename src/main/java/cn/edu/thu/tsfile/{spark => }/Executor.java (96%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/QueryProcessor.java (87%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/common/BasicOperator.java (92%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/common/FilterOperator.java (98%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/common/Operator.java (93%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/common/SQLConstant.java (99%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/common/SingleQuery.java (96%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/common/TSQueryPlan.java (96%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/exception/BasicOperatorException.java (88%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/exception/DNFOptimizeException.java (86%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/exception/LogicalOptimizeException.java (87%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/exception/MergeFilterException.java (86%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/exception/QueryOperatorException.java (81%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/exception/QueryProcessorException.java (90%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/exception/RemoveNotException.java (86%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/optimizer/DNFFilterOptimizer.java (94%) create mode 100755 src/main/java/cn/edu/thu/tsfile/optimizer/IFilterOptimizer.java rename src/main/java/cn/edu/thu/tsfile/{spark => }/optimizer/MergeSingleFilterOptimizer.java (96%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/optimizer/PhysicalOptimizer.java (98%) rename src/main/java/cn/edu/thu/tsfile/{spark => }/optimizer/RemoveNotOptimizer.java (84%) delete mode 100755 src/main/java/cn/edu/thu/tsfile/spark/optimizer/IFilterOptimizer.java rename src/main/scala/cn/edu/thu/tsfile/{spark => }/Converter.scala (98%) rename src/main/scala/cn/edu/thu/tsfile/{spark => }/DefaultSource.scala (98%) rename src/main/scala/cn/edu/thu/tsfile/{spark => }/package.scala (78%) rename src/test/java/cn/edu/thu/tsfile/{spark => }/CreateTSFile.java (99%) rename src/test/java/cn/edu/thu/tsfile/{spark => }/QueryPlanTest.java (88%) rename src/test/scala/cn/edu/thu/tsfile/{spark => }/ConverterTest.scala (98%) rename src/test/scala/cn/edu/thu/tsfile/{spark => }/TSFileSuit.scala (92%) diff --git a/README.md b/README.md index be37f0a..7853412 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,10 @@ The SparkSQL Table Structure is as follow: * **Example 2** ```scala - val spark = SparkSession.builder().master("local").getOrCreate() + import cn.edu.thu.tsfile.spark._ + import org.apache.spark.sql.SparkSession + + val spark = SparkSession.builder().master("local").getOrCreate() val df = spark.read .format("cn.edu.thu.tsfile.spark") .load("test.ts") @@ -136,6 +139,9 @@ The SparkSQL Table Structure is as follow: * **Example 3** ```scala + import cn.edu.thu.tsfile.spark._ + import org.apache.spark.sql.SparkSession + val spark = SparkSession.builder().master("local").getOrCreate() //create a table in SparkSQL and build relation with a TsFile diff --git a/src/main/java/cn/edu/thu/tsfile/spark/Executor.java b/src/main/java/cn/edu/thu/tsfile/Executor.java similarity index 96% rename from src/main/java/cn/edu/thu/tsfile/spark/Executor.java rename to src/main/java/cn/edu/thu/tsfile/Executor.java index baf6538..104c47d 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/Executor.java +++ b/src/main/java/cn/edu/thu/tsfile/Executor.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark; +package cn.edu.thu.tsfile; import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; import cn.edu.thu.tsfile.timeseries.read.query.QueryConfig; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/QueryProcessor.java b/src/main/java/cn/edu/thu/tsfile/QueryProcessor.java similarity index 87% rename from src/main/java/cn/edu/thu/tsfile/spark/QueryProcessor.java rename to src/main/java/cn/edu/thu/tsfile/QueryProcessor.java index 5e298f6..916f5af 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/QueryProcessor.java +++ b/src/main/java/cn/edu/thu/tsfile/QueryProcessor.java @@ -1,16 +1,16 @@ -package cn.edu.thu.tsfile.spark; +package cn.edu.thu.tsfile; +import cn.edu.thu.tsfile.common.FilterOperator; +import cn.edu.thu.tsfile.common.SQLConstant; +import cn.edu.thu.tsfile.common.SingleQuery; +import cn.edu.thu.tsfile.common.TSQueryPlan; import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; -import cn.edu.thu.tsfile.spark.common.FilterOperator; -import cn.edu.thu.tsfile.spark.common.SQLConstant; -import cn.edu.thu.tsfile.spark.common.SingleQuery; -import cn.edu.thu.tsfile.spark.common.TSQueryPlan; -import cn.edu.thu.tsfile.spark.exception.QueryOperatorException; -import cn.edu.thu.tsfile.spark.exception.QueryProcessorException; -import cn.edu.thu.tsfile.spark.optimizer.DNFFilterOptimizer; -import cn.edu.thu.tsfile.spark.optimizer.MergeSingleFilterOptimizer; -import cn.edu.thu.tsfile.spark.optimizer.PhysicalOptimizer; -import cn.edu.thu.tsfile.spark.optimizer.RemoveNotOptimizer; +import cn.edu.thu.tsfile.exception.QueryOperatorException; +import cn.edu.thu.tsfile.exception.QueryProcessorException; +import cn.edu.thu.tsfile.optimizer.DNFFilterOptimizer; +import cn.edu.thu.tsfile.optimizer.MergeSingleFilterOptimizer; +import cn.edu.thu.tsfile.optimizer.RemoveNotOptimizer; +import cn.edu.thu.tsfile.optimizer.PhysicalOptimizer; import java.io.IOException; import java.util.ArrayList; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/BasicOperator.java b/src/main/java/cn/edu/thu/tsfile/common/BasicOperator.java similarity index 92% rename from src/main/java/cn/edu/thu/tsfile/spark/common/BasicOperator.java rename to src/main/java/cn/edu/thu/tsfile/common/BasicOperator.java index 676cec1..cdc3d14 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/BasicOperator.java +++ b/src/main/java/cn/edu/thu/tsfile/common/BasicOperator.java @@ -1,7 +1,7 @@ -package cn.edu.thu.tsfile.spark.common; +package cn.edu.thu.tsfile.common; -import cn.edu.thu.tsfile.spark.exception.BasicOperatorException; +import cn.edu.thu.tsfile.exception.BasicOperatorException; /** * basic operators include < > >= <= !=. diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/FilterOperator.java b/src/main/java/cn/edu/thu/tsfile/common/FilterOperator.java similarity index 98% rename from src/main/java/cn/edu/thu/tsfile/spark/common/FilterOperator.java rename to src/main/java/cn/edu/thu/tsfile/common/FilterOperator.java index 907d7fe..2afa3dc 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/FilterOperator.java +++ b/src/main/java/cn/edu/thu/tsfile/common/FilterOperator.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.common; +package cn.edu.thu.tsfile.common; import cn.edu.thu.tsfile.timeseries.utils.StringContainer; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/Operator.java b/src/main/java/cn/edu/thu/tsfile/common/Operator.java similarity index 93% rename from src/main/java/cn/edu/thu/tsfile/spark/common/Operator.java rename to src/main/java/cn/edu/thu/tsfile/common/Operator.java index 3661719..fba9120 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/Operator.java +++ b/src/main/java/cn/edu/thu/tsfile/common/Operator.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.common; +package cn.edu.thu.tsfile.common; /** * This class is a superclass of all operator. diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/SQLConstant.java b/src/main/java/cn/edu/thu/tsfile/common/SQLConstant.java similarity index 99% rename from src/main/java/cn/edu/thu/tsfile/spark/common/SQLConstant.java rename to src/main/java/cn/edu/thu/tsfile/common/SQLConstant.java index a857db2..12b3f5b 100644 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/SQLConstant.java +++ b/src/main/java/cn/edu/thu/tsfile/common/SQLConstant.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.common; +package cn.edu.thu.tsfile.common; import cn.edu.thu.tsfile.timeseries.read.qp.Path; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/SingleQuery.java b/src/main/java/cn/edu/thu/tsfile/common/SingleQuery.java similarity index 96% rename from src/main/java/cn/edu/thu/tsfile/spark/common/SingleQuery.java rename to src/main/java/cn/edu/thu/tsfile/common/SingleQuery.java index 9e8225f..179639e 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/SingleQuery.java +++ b/src/main/java/cn/edu/thu/tsfile/common/SingleQuery.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.common; +package cn.edu.thu.tsfile.common; /** diff --git a/src/main/java/cn/edu/thu/tsfile/spark/common/TSQueryPlan.java b/src/main/java/cn/edu/thu/tsfile/common/TSQueryPlan.java similarity index 96% rename from src/main/java/cn/edu/thu/tsfile/spark/common/TSQueryPlan.java rename to src/main/java/cn/edu/thu/tsfile/common/TSQueryPlan.java index f154f4c..7e94f91 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/common/TSQueryPlan.java +++ b/src/main/java/cn/edu/thu/tsfile/common/TSQueryPlan.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.common; +package cn.edu.thu.tsfile.common; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/exception/BasicOperatorException.java b/src/main/java/cn/edu/thu/tsfile/exception/BasicOperatorException.java similarity index 88% rename from src/main/java/cn/edu/thu/tsfile/spark/exception/BasicOperatorException.java rename to src/main/java/cn/edu/thu/tsfile/exception/BasicOperatorException.java index e5a7634..4bed0bb 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/exception/BasicOperatorException.java +++ b/src/main/java/cn/edu/thu/tsfile/exception/BasicOperatorException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.exception; +package cn.edu.thu.tsfile.exception; /** diff --git a/src/main/java/cn/edu/thu/tsfile/spark/exception/DNFOptimizeException.java b/src/main/java/cn/edu/thu/tsfile/exception/DNFOptimizeException.java similarity index 86% rename from src/main/java/cn/edu/thu/tsfile/spark/exception/DNFOptimizeException.java rename to src/main/java/cn/edu/thu/tsfile/exception/DNFOptimizeException.java index 77db9ed..16a06bc 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/exception/DNFOptimizeException.java +++ b/src/main/java/cn/edu/thu/tsfile/exception/DNFOptimizeException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.exception; +package cn.edu.thu.tsfile.exception; /** diff --git a/src/main/java/cn/edu/thu/tsfile/spark/exception/LogicalOptimizeException.java b/src/main/java/cn/edu/thu/tsfile/exception/LogicalOptimizeException.java similarity index 87% rename from src/main/java/cn/edu/thu/tsfile/spark/exception/LogicalOptimizeException.java rename to src/main/java/cn/edu/thu/tsfile/exception/LogicalOptimizeException.java index cd768b0..ae36b08 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/exception/LogicalOptimizeException.java +++ b/src/main/java/cn/edu/thu/tsfile/exception/LogicalOptimizeException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.exception; +package cn.edu.thu.tsfile.exception; /** * This exception is threw whiling meeting error in logical optimizer process diff --git a/src/main/java/cn/edu/thu/tsfile/spark/exception/MergeFilterException.java b/src/main/java/cn/edu/thu/tsfile/exception/MergeFilterException.java similarity index 86% rename from src/main/java/cn/edu/thu/tsfile/spark/exception/MergeFilterException.java rename to src/main/java/cn/edu/thu/tsfile/exception/MergeFilterException.java index ce5a2d3..08674c2 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/exception/MergeFilterException.java +++ b/src/main/java/cn/edu/thu/tsfile/exception/MergeFilterException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.exception; +package cn.edu.thu.tsfile.exception; /** diff --git a/src/main/java/cn/edu/thu/tsfile/spark/exception/QueryOperatorException.java b/src/main/java/cn/edu/thu/tsfile/exception/QueryOperatorException.java similarity index 81% rename from src/main/java/cn/edu/thu/tsfile/spark/exception/QueryOperatorException.java rename to src/main/java/cn/edu/thu/tsfile/exception/QueryOperatorException.java index 31fcf09..bbfc264 100644 --- a/src/main/java/cn/edu/thu/tsfile/spark/exception/QueryOperatorException.java +++ b/src/main/java/cn/edu/thu/tsfile/exception/QueryOperatorException.java @@ -1,7 +1,5 @@ -package cn.edu.thu.tsfile.spark.exception; +package cn.edu.thu.tsfile.exception; -/** - */ public class QueryOperatorException extends LogicalOptimizeException { private static final long serialVersionUID = 8581594261924961899L; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/exception/QueryProcessorException.java b/src/main/java/cn/edu/thu/tsfile/exception/QueryProcessorException.java similarity index 90% rename from src/main/java/cn/edu/thu/tsfile/spark/exception/QueryProcessorException.java rename to src/main/java/cn/edu/thu/tsfile/exception/QueryProcessorException.java index efd11f5..d5371a1 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/exception/QueryProcessorException.java +++ b/src/main/java/cn/edu/thu/tsfile/exception/QueryProcessorException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.exception; +package cn.edu.thu.tsfile.exception; /** * This exception is threw whiling meeting error in query processor diff --git a/src/main/java/cn/edu/thu/tsfile/spark/exception/RemoveNotException.java b/src/main/java/cn/edu/thu/tsfile/exception/RemoveNotException.java similarity index 86% rename from src/main/java/cn/edu/thu/tsfile/spark/exception/RemoveNotException.java rename to src/main/java/cn/edu/thu/tsfile/exception/RemoveNotException.java index 30efb2f..b5bd438 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/exception/RemoveNotException.java +++ b/src/main/java/cn/edu/thu/tsfile/exception/RemoveNotException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark.exception; +package cn.edu.thu.tsfile.exception; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/DNFFilterOptimizer.java b/src/main/java/cn/edu/thu/tsfile/optimizer/DNFFilterOptimizer.java similarity index 94% rename from src/main/java/cn/edu/thu/tsfile/spark/optimizer/DNFFilterOptimizer.java rename to src/main/java/cn/edu/thu/tsfile/optimizer/DNFFilterOptimizer.java index d5bdb01..ce60714 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/DNFFilterOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/optimizer/DNFFilterOptimizer.java @@ -1,14 +1,14 @@ -package cn.edu.thu.tsfile.spark.optimizer; +package cn.edu.thu.tsfile.optimizer; -import cn.edu.thu.tsfile.spark.common.FilterOperator; -import cn.edu.thu.tsfile.spark.exception.DNFOptimizeException; +import cn.edu.thu.tsfile.common.FilterOperator; +import cn.edu.thu.tsfile.exception.DNFOptimizeException; import java.util.ArrayList; import java.util.List; -import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_AND; -import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_OR; +import static cn.edu.thu.tsfile.common.SQLConstant.KW_AND; +import static cn.edu.thu.tsfile.common.SQLConstant.KW_OR; public class DNFFilterOptimizer implements IFilterOptimizer { diff --git a/src/main/java/cn/edu/thu/tsfile/optimizer/IFilterOptimizer.java b/src/main/java/cn/edu/thu/tsfile/optimizer/IFilterOptimizer.java new file mode 100755 index 0000000..bc391f9 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfile/optimizer/IFilterOptimizer.java @@ -0,0 +1,16 @@ +package cn.edu.thu.tsfile.optimizer; + +import cn.edu.thu.tsfile.common.FilterOperator; +import cn.edu.thu.tsfile.exception.DNFOptimizeException; +import cn.edu.thu.tsfile.exception.MergeFilterException; +import cn.edu.thu.tsfile.exception.RemoveNotException; + +/** + * provide a filter operator, optimize it. + * + * @author kangrong + * + */ +public interface IFilterOptimizer { + FilterOperator optimize(FilterOperator filter) throws RemoveNotException, DNFOptimizeException, MergeFilterException; +} diff --git a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/MergeSingleFilterOptimizer.java b/src/main/java/cn/edu/thu/tsfile/optimizer/MergeSingleFilterOptimizer.java similarity index 96% rename from src/main/java/cn/edu/thu/tsfile/spark/optimizer/MergeSingleFilterOptimizer.java rename to src/main/java/cn/edu/thu/tsfile/optimizer/MergeSingleFilterOptimizer.java index fc94c74..783527e 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/MergeSingleFilterOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/optimizer/MergeSingleFilterOptimizer.java @@ -1,8 +1,8 @@ -package cn.edu.thu.tsfile.spark.optimizer; +package cn.edu.thu.tsfile.optimizer; -import cn.edu.thu.tsfile.spark.common.FilterOperator; -import cn.edu.thu.tsfile.spark.exception.MergeFilterException; +import cn.edu.thu.tsfile.exception.MergeFilterException; +import cn.edu.thu.tsfile.common.FilterOperator; import java.util.ArrayList; import java.util.Collections; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java b/src/main/java/cn/edu/thu/tsfile/optimizer/PhysicalOptimizer.java similarity index 98% rename from src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java rename to src/main/java/cn/edu/thu/tsfile/optimizer/PhysicalOptimizer.java index fffc958..cac5196 100644 --- a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/PhysicalOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/optimizer/PhysicalOptimizer.java @@ -1,7 +1,7 @@ -package cn.edu.thu.tsfile.spark.optimizer; +package cn.edu.thu.tsfile.optimizer; +import cn.edu.thu.tsfile.common.*; import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; -import cn.edu.thu.tsfile.spark.common.*; import cn.edu.thu.tsfile.timeseries.read.metadata.SeriesSchema; import cn.edu.thu.tsfile.timeseries.read.query.QueryEngine; diff --git a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/RemoveNotOptimizer.java b/src/main/java/cn/edu/thu/tsfile/optimizer/RemoveNotOptimizer.java similarity index 84% rename from src/main/java/cn/edu/thu/tsfile/spark/optimizer/RemoveNotOptimizer.java rename to src/main/java/cn/edu/thu/tsfile/optimizer/RemoveNotOptimizer.java index 2dc270a..aeb5f36 100755 --- a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/RemoveNotOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/optimizer/RemoveNotOptimizer.java @@ -1,19 +1,19 @@ -package cn.edu.thu.tsfile.spark.optimizer; +package cn.edu.thu.tsfile.optimizer; -import cn.edu.thu.tsfile.spark.common.BasicOperator; -import cn.edu.thu.tsfile.spark.common.FilterOperator; -import cn.edu.thu.tsfile.spark.common.SQLConstant; -import cn.edu.thu.tsfile.spark.exception.BasicOperatorException; -import cn.edu.thu.tsfile.spark.exception.RemoveNotException; +import cn.edu.thu.tsfile.common.BasicOperator; +import cn.edu.thu.tsfile.exception.BasicOperatorException; +import cn.edu.thu.tsfile.common.FilterOperator; +import cn.edu.thu.tsfile.common.SQLConstant; +import cn.edu.thu.tsfile.exception.RemoveNotException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; -import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_AND; -import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_NOT; -import static cn.edu.thu.tsfile.spark.common.SQLConstant.KW_OR; +import static cn.edu.thu.tsfile.common.SQLConstant.KW_AND; +import static cn.edu.thu.tsfile.common.SQLConstant.KW_NOT; +import static cn.edu.thu.tsfile.common.SQLConstant.KW_OR; public class RemoveNotOptimizer implements IFilterOptimizer { diff --git a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/IFilterOptimizer.java b/src/main/java/cn/edu/thu/tsfile/spark/optimizer/IFilterOptimizer.java deleted file mode 100755 index c2398bc..0000000 --- a/src/main/java/cn/edu/thu/tsfile/spark/optimizer/IFilterOptimizer.java +++ /dev/null @@ -1,16 +0,0 @@ -package cn.edu.thu.tsfile.spark.optimizer; - -import cn.edu.thu.tsfile.spark.common.FilterOperator; -import cn.edu.thu.tsfile.spark.exception.DNFOptimizeException; -import cn.edu.thu.tsfile.spark.exception.MergeFilterException; -import cn.edu.thu.tsfile.spark.exception.RemoveNotException; - -/** - * provide a filter operator, optimize it. - * - * @author kangrong - * - */ -public interface IFilterOptimizer { - FilterOperator optimize(FilterOperator filter) throws RemoveNotException, DNFOptimizeException, MergeFilterException; -} diff --git a/src/main/scala/cn/edu/thu/tsfile/spark/Converter.scala b/src/main/scala/cn/edu/thu/tsfile/Converter.scala similarity index 98% rename from src/main/scala/cn/edu/thu/tsfile/spark/Converter.scala rename to src/main/scala/cn/edu/thu/tsfile/Converter.scala index 0f8c3ee..9c55558 100755 --- a/src/main/scala/cn/edu/thu/tsfile/spark/Converter.scala +++ b/src/main/scala/cn/edu/thu/tsfile/Converter.scala @@ -1,11 +1,11 @@ -package cn.edu.thu.tsfile.spark +package cn.edu.thu.tsfile import java.util +import cn.edu.thu.tsfile.common.{BasicOperator, FilterOperator, SQLConstant, TSQueryPlan} import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader import cn.edu.thu.tsfile.file.metadata.enums.TSDataType import cn.edu.thu.tsfile.hadoop.io.HDFSInputStream -import cn.edu.thu.tsfile.spark.common.{BasicOperator, FilterOperator, SQLConstant, TSQueryPlan} import cn.edu.thu.tsfile.timeseries.read.metadata.SeriesSchema import cn.edu.thu.tsfile.timeseries.read.query.{QueryConfig, QueryEngine} import cn.edu.thu.tsfile.timeseries.read.readSupport.Field diff --git a/src/main/scala/cn/edu/thu/tsfile/spark/DefaultSource.scala b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala similarity index 98% rename from src/main/scala/cn/edu/thu/tsfile/spark/DefaultSource.scala rename to src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala index a06e714..badad8e 100755 --- a/src/main/scala/cn/edu/thu/tsfile/spark/DefaultSource.scala +++ b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala @@ -1,13 +1,13 @@ -package cn.edu.thu.tsfile.spark +package cn.edu.thu.tsfile import java.io._ import java.net.URI import java.util +import cn.edu.thu.tsfile.common.SQLConstant import cn.edu.thu.tsfile.common.constant.QueryConstant import cn.edu.thu.tsfile.hadoop.io.HDFSInputStream -import cn.edu.thu.tsfile.spark.DefaultSource.SerializableConfiguration -import cn.edu.thu.tsfile.spark.common.SQLConstant +import cn.edu.thu.tsfile.DefaultSource.SerializableConfiguration import cn.edu.thu.tsfile.timeseries.read.query.QueryDataSet import cn.edu.thu.tsfile.timeseries.read.readSupport.{Field, RowRecord} import com.esotericsoftware.kryo.io.{Input, Output} diff --git a/src/main/scala/cn/edu/thu/tsfile/spark/package.scala b/src/main/scala/cn/edu/thu/tsfile/package.scala similarity index 78% rename from src/main/scala/cn/edu/thu/tsfile/spark/package.scala rename to src/main/scala/cn/edu/thu/tsfile/package.scala index 15e0007..f1e6ba7 100755 --- a/src/main/scala/cn/edu/thu/tsfile/spark/package.scala +++ b/src/main/scala/cn/edu/thu/tsfile/package.scala @@ -1,13 +1,13 @@ -package cn.edu.thu.tsfile +package cn.edu.thu import org.apache.spark.sql.{DataFrame, DataFrameReader} -package object spark { +package object tsfile { /** * add a method to DataFrameReader */ implicit class TSFileDataFrameReader(reader: DataFrameReader) { - def tsfile: String => DataFrame = reader.format("cn.edu.thu.tsfile.spark").load + def tsfile: String => DataFrame = reader.format("cn.edu.thu.tsfile").load } } diff --git a/src/test/java/cn/edu/thu/tsfile/spark/CreateTSFile.java b/src/test/java/cn/edu/thu/tsfile/CreateTSFile.java similarity index 99% rename from src/test/java/cn/edu/thu/tsfile/spark/CreateTSFile.java rename to src/test/java/cn/edu/thu/tsfile/CreateTSFile.java index 812090b..119cf25 100755 --- a/src/test/java/cn/edu/thu/tsfile/spark/CreateTSFile.java +++ b/src/test/java/cn/edu/thu/tsfile/CreateTSFile.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.spark; +package cn.edu.thu.tsfile; import cn.edu.thu.tsfile.common.conf.TSFileConfig; import cn.edu.thu.tsfile.common.conf.TSFileDescriptor; diff --git a/src/test/java/cn/edu/thu/tsfile/spark/QueryPlanTest.java b/src/test/java/cn/edu/thu/tsfile/QueryPlanTest.java similarity index 88% rename from src/test/java/cn/edu/thu/tsfile/spark/QueryPlanTest.java rename to src/test/java/cn/edu/thu/tsfile/QueryPlanTest.java index 470d650..f2c34d4 100644 --- a/src/test/java/cn/edu/thu/tsfile/spark/QueryPlanTest.java +++ b/src/test/java/cn/edu/thu/tsfile/QueryPlanTest.java @@ -1,11 +1,11 @@ -package cn.edu.thu.tsfile.spark; +package cn.edu.thu.tsfile; -import cn.edu.thu.tsfile.spark.common.BasicOperator; -import cn.edu.thu.tsfile.spark.common.FilterOperator; -import cn.edu.thu.tsfile.spark.common.SQLConstant; -import cn.edu.thu.tsfile.spark.common.TSQueryPlan; -import cn.edu.thu.tsfile.spark.exception.QueryProcessorException; +import cn.edu.thu.tsfile.common.BasicOperator; +import cn.edu.thu.tsfile.common.TSQueryPlan; +import cn.edu.thu.tsfile.common.FilterOperator; +import cn.edu.thu.tsfile.common.SQLConstant; +import cn.edu.thu.tsfile.exception.QueryProcessorException; import cn.edu.thu.tsfile.timeseries.read.LocalFileInput; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/scala/cn/edu/thu/tsfile/spark/ConverterTest.scala b/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala similarity index 98% rename from src/test/scala/cn/edu/thu/tsfile/spark/ConverterTest.scala rename to src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala index 84f407e..773ffef 100644 --- a/src/test/scala/cn/edu/thu/tsfile/spark/ConverterTest.scala +++ b/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala @@ -1,10 +1,10 @@ -package cn.edu.thu.tsfile.spark +package cn.edu.thu.tsfile import java.io.File import java.util +import cn.edu.thu.tsfile.common.SQLConstant import cn.edu.thu.tsfile.file.metadata.enums.{TSDataType, TSEncoding} -import cn.edu.thu.tsfile.spark.common.SQLConstant import cn.edu.thu.tsfile.timeseries.read.LocalFileInput import cn.edu.thu.tsfile.timeseries.read.metadata.SeriesSchema import cn.edu.thu.tsfile.timeseries.read.query.QueryConfig diff --git a/src/test/scala/cn/edu/thu/tsfile/spark/TSFileSuit.scala b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala similarity index 92% rename from src/test/scala/cn/edu/thu/tsfile/spark/TSFileSuit.scala rename to src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala index de43cf4..aac3fda 100644 --- a/src/test/scala/cn/edu/thu/tsfile/spark/TSFileSuit.scala +++ b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala @@ -1,8 +1,8 @@ -package cn.edu.thu.tsfile.spark +package cn.edu.thu.tsfile import java.io.File -import cn.edu.thu.tsfile.spark.common.SQLConstant +import cn.edu.thu.tsfile.common.SQLConstant import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types._ import org.junit.Assert @@ -45,21 +45,21 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { } test("qp") { - val df = spark.read.format("cn.edu.thu.tsfile.spark").load(tsfileFolder) + val df = spark.read.format("cn.edu.thu.tsfile").load(tsfileFolder) df.createOrReplaceTempView("tsfile_table") val newDf = spark.sql("select s1,s2 from tsfile_table where delta_object = 'root.car.d1' and time <= 10 and (time > 5 or s1 > 10)") Assert.assertEquals(0, newDf.count()) } test("testMultiFilesNoneExistDelta_object") { - val df = spark.read.format("cn.edu.thu.tsfile.spark").load(tsfileFolder) + val df = spark.read.format("cn.edu.thu.tsfile").load(tsfileFolder) df.createOrReplaceTempView("tsfile_table") val newDf = spark.sql("select * from tsfile_table where delta_object = 'd4'") Assert.assertEquals(0, newDf.count()) } test("testMultiFilesWithFilterOr") { - val df = spark.read.format("cn.edu.thu.tsfile.spark").load(tsfileFolder) + val df = spark.read.format("cn.edu.thu.tsfile").load(tsfileFolder) df.createOrReplaceTempView("tsfile_table") val newDf = spark.sql("select * from tsfile_table where s1 < 2 or s2 > 60") Assert.assertEquals(4, newDf.count()) From 96fe152f2eb18c9f025680b9c7bf2eae806054a1 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Tue, 13 Jun 2017 21:18:20 +0800 Subject: [PATCH 05/15] fix pom bug --- README.md | 32 ++-- pom.xml | 151 +++++++++++------- .../cn/edu/thu/tsfile/io}/CreateTSFile.java | 2 +- .../{hadoop => }/io/HDFSInputStream.java | 19 +-- .../{hadoop => }/io/HDFSOutputStream.java | 2 +- .../tsfile/optimizer/IFilterOptimizer.java | 16 -- .../cn/edu/thu/tsfile/{ => qp}/Executor.java | 2 +- .../thu/tsfile/{ => qp}/QueryProcessor.java | 22 +-- .../tsfile/{ => qp}/common/BasicOperator.java | 4 +- .../{ => qp}/common/FilterOperator.java | 2 +- .../thu/tsfile/{ => qp}/common/Operator.java | 2 +- .../tsfile/{ => qp}/common/SQLConstant.java | 2 +- .../tsfile/{ => qp}/common/SingleQuery.java | 2 +- .../tsfile/{ => qp}/common/TSQueryPlan.java | 4 +- .../exception/BasicOperatorException.java | 2 +- .../exception/DNFOptimizeException.java | 2 +- .../exception/LogicalOptimizeException.java | 2 +- .../exception/MergeFilterException.java | 6 +- .../exception/QueryOperatorException.java | 2 +- .../exception/QueryProcessorException.java | 2 +- .../exception/RemoveNotException.java | 2 +- .../optimizer/DNFFilterOptimizer.java | 11 +- .../tsfile/qp/optimizer/IFilterOptimizer.java | 16 ++ .../optimizer/MergeSingleFilterOptimizer.java | 7 +- .../{ => qp}/optimizer/PhysicalOptimizer.java | 4 +- .../optimizer/RemoveNotOptimizer.java | 18 +-- .../scala/cn/edu/thu/tsfile/Converter.scala | 5 +- .../cn/edu/thu/tsfile/DefaultSource.scala | 21 +-- .../java/cn/edu/thu/tsfile/QueryPlanTest.java | 60 ------- .../hadoop/io/HDFSInputOutputStreamTest.java | 67 -------- src/test/resources/test.tsfile | Bin 1233 -> 0 bytes .../cn/edu/thu/tsfile/ConverterTest.scala | 5 +- .../scala/cn/edu/thu/tsfile/TSFileSuit.scala | 26 ++- 33 files changed, 220 insertions(+), 300 deletions(-) rename src/{test/java/cn/edu/thu/tsfile => main/java/cn/edu/thu/tsfile/io}/CreateTSFile.java (99%) rename src/main/java/cn/edu/thu/tsfile/{hadoop => }/io/HDFSInputStream.java (79%) rename src/main/java/cn/edu/thu/tsfile/{hadoop => }/io/HDFSOutputStream.java (98%) delete mode 100755 src/main/java/cn/edu/thu/tsfile/optimizer/IFilterOptimizer.java rename src/main/java/cn/edu/thu/tsfile/{ => qp}/Executor.java (97%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/QueryProcessor.java (88%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/common/BasicOperator.java (92%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/common/FilterOperator.java (99%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/common/Operator.java (93%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/common/SQLConstant.java (99%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/common/SingleQuery.java (96%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/common/TSQueryPlan.java (89%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/exception/BasicOperatorException.java (88%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/exception/DNFOptimizeException.java (87%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/exception/LogicalOptimizeException.java (88%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/exception/MergeFilterException.java (68%) mode change 100755 => 100644 rename src/main/java/cn/edu/thu/tsfile/{ => qp}/exception/QueryOperatorException.java (84%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/exception/QueryProcessorException.java (91%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/exception/RemoveNotException.java (87%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/optimizer/DNFFilterOptimizer.java (94%) create mode 100755 src/main/java/cn/edu/thu/tsfile/qp/optimizer/IFilterOptimizer.java rename src/main/java/cn/edu/thu/tsfile/{ => qp}/optimizer/MergeSingleFilterOptimizer.java (96%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/optimizer/PhysicalOptimizer.java (98%) rename src/main/java/cn/edu/thu/tsfile/{ => qp}/optimizer/RemoveNotOptimizer.java (85%) delete mode 100644 src/test/java/cn/edu/thu/tsfile/QueryPlanTest.java delete mode 100644 src/test/java/cn/edu/thu/tsfile/hadoop/io/HDFSInputOutputStreamTest.java delete mode 100644 src/test/resources/test.tsfile diff --git a/README.md b/README.md index 7853412..f395460 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ -# tsfile-spark-connector +# cn.edu.thu.tsfile-spark-connector 将一个或多个TsFile展示成SparkSQL中的一张表。允许指定单个目录，或使用通配符匹配多个目录。如果是多个TsFile，schema将保留各个TsFile中sensor的并集。 ## 示例 -src/test/scala/cn.edu.thu.tsfile.spark.TSFileSuit +src/test/scala/cn.edu.thu.cn.edu.thu.tsfile.spark.TSFileSuit ## 路径指定方式 -basefolder/key=1/file1.tsfile +basefolder/key=1/file1.cn.edu.thu.tsfile -basefolder/key=2/file2.tsfile +basefolder/key=2/file2.cn.edu.thu.tsfile 指定basefolder为path，会在表中多加一列key，值为1或2。 如： @@ -23,11 +23,11 @@ path=basefolder 如果使用通配符指定，将不会当做partiton 如： -path=basefolder/\*/\*.tsfile +path=basefolder/\*/\*.cn.edu.thu.tsfile -basefolder/file1.tsfile -basefolder/file2.tsfile +basefolder/file1.cn.edu.thu.tsfile +basefolder/file2.cn.edu.thu.tsfile 指定basefolder会将多个tsfile的schema合并，保留sensor的并集 @@ -104,13 +104,13 @@ The SparkSQL Table Structure is as follow: ```scala // import this library and Spark - import cn.edu.thu.tsfile.spark._ + import cn.edu.thu.cn.edu.thu.tsfile._ import org.apache.spark.sql.SparkSession val spark = SparkSession.builder().master("local").getOrCreate() //read data in TsFile and create a table - val df = spark.read.tsfile("test.ts") + val df = spark.read.cn.edu.thu.tsfile("test.ts") df.createOrReplaceTempView("TsFile_table") //query with filter @@ -123,12 +123,12 @@ The SparkSQL Table Structure is as follow: * **Example 2** ```scala - import cn.edu.thu.tsfile.spark._ + import cn.edu.thu.cn.edu.thu.tsfile._ import org.apache.spark.sql.SparkSession val spark = SparkSession.builder().master("local").getOrCreate() val df = spark.read - .format("cn.edu.thu.tsfile.spark") + .format("cn.edu.thu.cn.edu.thu.tsfile.spark") .load("test.ts") @@ -139,13 +139,13 @@ The SparkSQL Table Structure is as follow: * **Example 3** ```scala - import cn.edu.thu.tsfile.spark._ + import cn.edu.thu.cn.edu.thu.tsfile._ import org.apache.spark.sql.SparkSession val spark = SparkSession.builder().master("local").getOrCreate() //create a table in SparkSQL and build relation with a TsFile - spark.sql("create temporary view TsFile using cn.edu.thu.tsfile.spark options(path = \"test.ts\")") + spark.sql("create temporary view TsFile using cn.edu.thu.cn.edu.thu.tsfile.spark options(path = \"test.ts\")") spark.sql("select * from TsFile where sensor_1 > 1.2").show() @@ -159,13 +159,13 @@ The SparkSQL Table Structure is as follow: mvn package -DskipTests 包所在位置： -/tsfile-kmx-spark-connector/target/tsfile-1.0-SNAPSHOT-jar-with-dependencies.jar +/cn.edu.thu.tsfile-kmx-spark-connector/target/cn.edu.thu.tsfile-1.0-SNAPSHOT-jar-with-dependencies.jar ``` ``` -$ bin/spark-shell --jars tsfile-spark-0.1.0-jar-with-dependencies.jar +$ bin/spark-shell --jars cn.edu.thu.tsfile-spark-0.1.0-jar-with-dependencies.jar -scala> sql("CREATE TEMPORARY TABLE TsFile_table USING cn.edu.thu.tsfile.spark OPTIONS (path \"hdfs://localhost:9000/test.ts\")") +scala> sql("CREATE TEMPORARY TABLE TsFile_table USING cn.edu.thu.cn.edu.thu.tsfile.spark OPTIONS (path \"hdfs://localhost:9000/test.ts\")") scala> sql("select * from TsFile_table where sensor_1 > 1.2").show() ``` \ No newline at end of file diff --git a/pom.xml b/pom.xml index eb473aa..5634f5b 100644 --- a/pom.xml +++ b/pom.xml @@ -1,61 +1,102 @@ - 4.0.0 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - cn.edu.thu - tsfile-spark-connector - 0.1.0 - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.8 - 1.8 - - - - - jar + cn.edu.thu + tsfile-spark-connector + 0.1.0 + jar - tsfile - http://maven.apache.org + + UTF-8 + 2.7.3 + 1.8 + 2.0.1 + 4.12 + + + + + junit + junit + ${junit.version} + test + + + cn.edu.thu + tsfile + 0.1.0 + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + org.apache.spark + spark-core_2.11 + ${spark.version} + provided + + + org.apache.spark + spark-sql_2.11 + ${spark.version} + provided + + + org.scala-lang + scala-library + 2.11.8 + + + org.scalatest + scalatest_2.11 + 3.0.0 + test + + + + + + + net.alchim31.maven + scala-maven-plugin + 3.2.0 + + + compile-scala + compile + + add-source + compile + + + + test-compile-scala + test-compile + + add-source + testCompile + + + + + 2.11.8 + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${compile.version} + ${compile.version} + ${project.build.sourceEncoding} + + + + - - UTF-8 - 2.7.3 - 4.12 - - - - junit - junit - ${junit.version} - test - - - cn.edu.thu - tsfile - 0.1.0 - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - - - org.apache.spark - spark-sql_2.11 - 2.0.1 - provided - - - org.scalatest - scalatest_2.11 - 3.0.0 - test - - diff --git a/src/test/java/cn/edu/thu/tsfile/CreateTSFile.java b/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java similarity index 99% rename from src/test/java/cn/edu/thu/tsfile/CreateTSFile.java rename to src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java index 119cf25..caa57be 100755 --- a/src/test/java/cn/edu/thu/tsfile/CreateTSFile.java +++ b/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile; +package cn.edu.thu.tsfile.io; import cn.edu.thu.tsfile.common.conf.TSFileConfig; import cn.edu.thu.tsfile.common.conf.TSFileDescriptor; diff --git a/src/main/java/cn/edu/thu/tsfile/hadoop/io/HDFSInputStream.java b/src/main/java/cn/edu/thu/tsfile/io/HDFSInputStream.java similarity index 79% rename from src/main/java/cn/edu/thu/tsfile/hadoop/io/HDFSInputStream.java rename to src/main/java/cn/edu/thu/tsfile/io/HDFSInputStream.java index 9180781..8ee01e9 100644 --- a/src/main/java/cn/edu/thu/tsfile/hadoop/io/HDFSInputStream.java +++ b/src/main/java/cn/edu/thu/tsfile/io/HDFSInputStream.java @@ -1,11 +1,8 @@ -package cn.edu.thu.tsfile.hadoop.io; - +package cn.edu.thu.tsfile.io; import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.IOException; @@ -18,8 +15,8 @@ */ public class HDFSInputStream implements TSRandomAccessFileReader { - private FSDataInputStream fsDataInputStream; - private FileStatus fileStatus; + private final FSDataInputStream fsDataInputStream; + private final long length; public HDFSInputStream(String filePath) throws IOException { @@ -31,11 +28,9 @@ public HDFSInputStream(String filePath, Configuration configuration) throws IOEx this(new Path(filePath),configuration); } - public HDFSInputStream(Path path, Configuration configuration) throws IOException { - - FileSystem fs = FileSystem.get(configuration); - fsDataInputStream = fs.open(path); - fileStatus = fs.getFileStatus(path); + public HDFSInputStream(Path path, Configuration conf) throws IOException { + length = path.getFileSystem(conf).getFileStatus(path).getLen(); + fsDataInputStream = path.getFileSystem(conf).open(path); } public void seek(long offset) throws IOException { @@ -50,7 +45,7 @@ public int read() throws IOException { public long length() throws IOException { - return fileStatus.getLen(); + return length; } public int readInt() throws IOException { diff --git a/src/main/java/cn/edu/thu/tsfile/hadoop/io/HDFSOutputStream.java b/src/main/java/cn/edu/thu/tsfile/io/HDFSOutputStream.java similarity index 98% rename from src/main/java/cn/edu/thu/tsfile/hadoop/io/HDFSOutputStream.java rename to src/main/java/cn/edu/thu/tsfile/io/HDFSOutputStream.java index 57addc8..272b5f9 100644 --- a/src/main/java/cn/edu/thu/tsfile/hadoop/io/HDFSOutputStream.java +++ b/src/main/java/cn/edu/thu/tsfile/io/HDFSOutputStream.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.hadoop.io; +package cn.edu.thu.tsfile.io; import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter; import org.apache.hadoop.conf.Configuration; diff --git a/src/main/java/cn/edu/thu/tsfile/optimizer/IFilterOptimizer.java b/src/main/java/cn/edu/thu/tsfile/optimizer/IFilterOptimizer.java deleted file mode 100755 index bc391f9..0000000 --- a/src/main/java/cn/edu/thu/tsfile/optimizer/IFilterOptimizer.java +++ /dev/null @@ -1,16 +0,0 @@ -package cn.edu.thu.tsfile.optimizer; - -import cn.edu.thu.tsfile.common.FilterOperator; -import cn.edu.thu.tsfile.exception.DNFOptimizeException; -import cn.edu.thu.tsfile.exception.MergeFilterException; -import cn.edu.thu.tsfile.exception.RemoveNotException; - -/** - * provide a filter operator, optimize it. - * - * @author kangrong - * - */ -public interface IFilterOptimizer { - FilterOperator optimize(FilterOperator filter) throws RemoveNotException, DNFOptimizeException, MergeFilterException; -} diff --git a/src/main/java/cn/edu/thu/tsfile/Executor.java b/src/main/java/cn/edu/thu/tsfile/qp/Executor.java similarity index 97% rename from src/main/java/cn/edu/thu/tsfile/Executor.java rename to src/main/java/cn/edu/thu/tsfile/qp/Executor.java index 104c47d..7033579 100755 --- a/src/main/java/cn/edu/thu/tsfile/Executor.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/Executor.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile; +package cn.edu.thu.tsfile.qp; import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; import cn.edu.thu.tsfile.timeseries.read.query.QueryConfig; diff --git a/src/main/java/cn/edu/thu/tsfile/QueryProcessor.java b/src/main/java/cn/edu/thu/tsfile/qp/QueryProcessor.java similarity index 88% rename from src/main/java/cn/edu/thu/tsfile/QueryProcessor.java rename to src/main/java/cn/edu/thu/tsfile/qp/QueryProcessor.java index 916f5af..5eb8357 100755 --- a/src/main/java/cn/edu/thu/tsfile/QueryProcessor.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/QueryProcessor.java @@ -1,16 +1,16 @@ -package cn.edu.thu.tsfile; +package cn.edu.thu.tsfile.qp; -import cn.edu.thu.tsfile.common.FilterOperator; -import cn.edu.thu.tsfile.common.SQLConstant; -import cn.edu.thu.tsfile.common.SingleQuery; -import cn.edu.thu.tsfile.common.TSQueryPlan; import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; -import cn.edu.thu.tsfile.exception.QueryOperatorException; -import cn.edu.thu.tsfile.exception.QueryProcessorException; -import cn.edu.thu.tsfile.optimizer.DNFFilterOptimizer; -import cn.edu.thu.tsfile.optimizer.MergeSingleFilterOptimizer; -import cn.edu.thu.tsfile.optimizer.RemoveNotOptimizer; -import cn.edu.thu.tsfile.optimizer.PhysicalOptimizer; +import cn.edu.thu.tsfile.qp.common.FilterOperator; +import cn.edu.thu.tsfile.qp.common.SQLConstant; +import cn.edu.thu.tsfile.qp.common.SingleQuery; +import cn.edu.thu.tsfile.qp.common.TSQueryPlan; +import cn.edu.thu.tsfile.qp.exception.QueryOperatorException; +import cn.edu.thu.tsfile.qp.exception.QueryProcessorException; +import cn.edu.thu.tsfile.qp.optimizer.DNFFilterOptimizer; +import cn.edu.thu.tsfile.qp.optimizer.MergeSingleFilterOptimizer; +import cn.edu.thu.tsfile.qp.optimizer.PhysicalOptimizer; +import cn.edu.thu.tsfile.qp.optimizer.RemoveNotOptimizer; import java.io.IOException; import java.util.ArrayList; diff --git a/src/main/java/cn/edu/thu/tsfile/common/BasicOperator.java b/src/main/java/cn/edu/thu/tsfile/qp/common/BasicOperator.java similarity index 92% rename from src/main/java/cn/edu/thu/tsfile/common/BasicOperator.java rename to src/main/java/cn/edu/thu/tsfile/qp/common/BasicOperator.java index cdc3d14..82e108b 100755 --- a/src/main/java/cn/edu/thu/tsfile/common/BasicOperator.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/common/BasicOperator.java @@ -1,7 +1,7 @@ -package cn.edu.thu.tsfile.common; +package cn.edu.thu.tsfile.qp.common; -import cn.edu.thu.tsfile.exception.BasicOperatorException; +import cn.edu.thu.tsfile.qp.exception.BasicOperatorException; /** * basic operators include < > >= <= !=. diff --git a/src/main/java/cn/edu/thu/tsfile/common/FilterOperator.java b/src/main/java/cn/edu/thu/tsfile/qp/common/FilterOperator.java similarity index 99% rename from src/main/java/cn/edu/thu/tsfile/common/FilterOperator.java rename to src/main/java/cn/edu/thu/tsfile/qp/common/FilterOperator.java index 2afa3dc..36fa931 100755 --- a/src/main/java/cn/edu/thu/tsfile/common/FilterOperator.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/common/FilterOperator.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.common; +package cn.edu.thu.tsfile.qp.common; import cn.edu.thu.tsfile.timeseries.utils.StringContainer; diff --git a/src/main/java/cn/edu/thu/tsfile/common/Operator.java b/src/main/java/cn/edu/thu/tsfile/qp/common/Operator.java similarity index 93% rename from src/main/java/cn/edu/thu/tsfile/common/Operator.java rename to src/main/java/cn/edu/thu/tsfile/qp/common/Operator.java index fba9120..2a53125 100755 --- a/src/main/java/cn/edu/thu/tsfile/common/Operator.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/common/Operator.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.common; +package cn.edu.thu.tsfile.qp.common; /** * This class is a superclass of all operator. diff --git a/src/main/java/cn/edu/thu/tsfile/common/SQLConstant.java b/src/main/java/cn/edu/thu/tsfile/qp/common/SQLConstant.java similarity index 99% rename from src/main/java/cn/edu/thu/tsfile/common/SQLConstant.java rename to src/main/java/cn/edu/thu/tsfile/qp/common/SQLConstant.java index 12b3f5b..7b4b0b2 100644 --- a/src/main/java/cn/edu/thu/tsfile/common/SQLConstant.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/common/SQLConstant.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.common; +package cn.edu.thu.tsfile.qp.common; import cn.edu.thu.tsfile.timeseries.read.qp.Path; diff --git a/src/main/java/cn/edu/thu/tsfile/common/SingleQuery.java b/src/main/java/cn/edu/thu/tsfile/qp/common/SingleQuery.java similarity index 96% rename from src/main/java/cn/edu/thu/tsfile/common/SingleQuery.java rename to src/main/java/cn/edu/thu/tsfile/qp/common/SingleQuery.java index 179639e..479c7cd 100755 --- a/src/main/java/cn/edu/thu/tsfile/common/SingleQuery.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/common/SingleQuery.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.common; +package cn.edu.thu.tsfile.qp.common; /** diff --git a/src/main/java/cn/edu/thu/tsfile/common/TSQueryPlan.java b/src/main/java/cn/edu/thu/tsfile/qp/common/TSQueryPlan.java similarity index 89% rename from src/main/java/cn/edu/thu/tsfile/common/TSQueryPlan.java rename to src/main/java/cn/edu/thu/tsfile/qp/common/TSQueryPlan.java index 7e94f91..0c60e27 100755 --- a/src/main/java/cn/edu/thu/tsfile/common/TSQueryPlan.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/common/TSQueryPlan.java @@ -1,10 +1,10 @@ -package cn.edu.thu.tsfile.common; +package cn.edu.thu.tsfile.qp.common; import java.util.ArrayList; import java.util.List; /** - * One tsfile logical query plan that can be performed at one time + * One cn.edu.thu.tsfile logical query plan that can be performed at one time * */ public class TSQueryPlan { diff --git a/src/main/java/cn/edu/thu/tsfile/exception/BasicOperatorException.java b/src/main/java/cn/edu/thu/tsfile/qp/exception/BasicOperatorException.java similarity index 88% rename from src/main/java/cn/edu/thu/tsfile/exception/BasicOperatorException.java rename to src/main/java/cn/edu/thu/tsfile/qp/exception/BasicOperatorException.java index 4bed0bb..3caebbc 100755 --- a/src/main/java/cn/edu/thu/tsfile/exception/BasicOperatorException.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/exception/BasicOperatorException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.exception; +package cn.edu.thu.tsfile.qp.exception; /** diff --git a/src/main/java/cn/edu/thu/tsfile/exception/DNFOptimizeException.java b/src/main/java/cn/edu/thu/tsfile/qp/exception/DNFOptimizeException.java similarity index 87% rename from src/main/java/cn/edu/thu/tsfile/exception/DNFOptimizeException.java rename to src/main/java/cn/edu/thu/tsfile/qp/exception/DNFOptimizeException.java index 16a06bc..df027a4 100755 --- a/src/main/java/cn/edu/thu/tsfile/exception/DNFOptimizeException.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/exception/DNFOptimizeException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.exception; +package cn.edu.thu.tsfile.qp.exception; /** diff --git a/src/main/java/cn/edu/thu/tsfile/exception/LogicalOptimizeException.java b/src/main/java/cn/edu/thu/tsfile/qp/exception/LogicalOptimizeException.java similarity index 88% rename from src/main/java/cn/edu/thu/tsfile/exception/LogicalOptimizeException.java rename to src/main/java/cn/edu/thu/tsfile/qp/exception/LogicalOptimizeException.java index ae36b08..d73b873 100755 --- a/src/main/java/cn/edu/thu/tsfile/exception/LogicalOptimizeException.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/exception/LogicalOptimizeException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.exception; +package cn.edu.thu.tsfile.qp.exception; /** * This exception is threw whiling meeting error in logical optimizer process diff --git a/src/main/java/cn/edu/thu/tsfile/exception/MergeFilterException.java b/src/main/java/cn/edu/thu/tsfile/qp/exception/MergeFilterException.java old mode 100755 new mode 100644 similarity index 68% rename from src/main/java/cn/edu/thu/tsfile/exception/MergeFilterException.java rename to src/main/java/cn/edu/thu/tsfile/qp/exception/MergeFilterException.java index 08674c2..03f6d83 --- a/src/main/java/cn/edu/thu/tsfile/exception/MergeFilterException.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/exception/MergeFilterException.java @@ -1,10 +1,6 @@ -package cn.edu.thu.tsfile.exception; +package cn.edu.thu.tsfile.qp.exception; -/** - * This exception is threw whiling meeting error in - * - */ public class MergeFilterException extends LogicalOptimizeException { private static final long serialVersionUID = 8581594261924961899L; diff --git a/src/main/java/cn/edu/thu/tsfile/exception/QueryOperatorException.java b/src/main/java/cn/edu/thu/tsfile/qp/exception/QueryOperatorException.java similarity index 84% rename from src/main/java/cn/edu/thu/tsfile/exception/QueryOperatorException.java rename to src/main/java/cn/edu/thu/tsfile/qp/exception/QueryOperatorException.java index bbfc264..5613b1b 100644 --- a/src/main/java/cn/edu/thu/tsfile/exception/QueryOperatorException.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/exception/QueryOperatorException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.exception; +package cn.edu.thu.tsfile.qp.exception; public class QueryOperatorException extends LogicalOptimizeException { diff --git a/src/main/java/cn/edu/thu/tsfile/exception/QueryProcessorException.java b/src/main/java/cn/edu/thu/tsfile/qp/exception/QueryProcessorException.java similarity index 91% rename from src/main/java/cn/edu/thu/tsfile/exception/QueryProcessorException.java rename to src/main/java/cn/edu/thu/tsfile/qp/exception/QueryProcessorException.java index d5371a1..503b5c4 100755 --- a/src/main/java/cn/edu/thu/tsfile/exception/QueryProcessorException.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/exception/QueryProcessorException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.exception; +package cn.edu.thu.tsfile.qp.exception; /** * This exception is threw whiling meeting error in query processor diff --git a/src/main/java/cn/edu/thu/tsfile/exception/RemoveNotException.java b/src/main/java/cn/edu/thu/tsfile/qp/exception/RemoveNotException.java similarity index 87% rename from src/main/java/cn/edu/thu/tsfile/exception/RemoveNotException.java rename to src/main/java/cn/edu/thu/tsfile/qp/exception/RemoveNotException.java index b5bd438..0d36043 100755 --- a/src/main/java/cn/edu/thu/tsfile/exception/RemoveNotException.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/exception/RemoveNotException.java @@ -1,4 +1,4 @@ -package cn.edu.thu.tsfile.exception; +package cn.edu.thu.tsfile.qp.exception; diff --git a/src/main/java/cn/edu/thu/tsfile/optimizer/DNFFilterOptimizer.java b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/DNFFilterOptimizer.java similarity index 94% rename from src/main/java/cn/edu/thu/tsfile/optimizer/DNFFilterOptimizer.java rename to src/main/java/cn/edu/thu/tsfile/qp/optimizer/DNFFilterOptimizer.java index ce60714..aeb42e1 100755 --- a/src/main/java/cn/edu/thu/tsfile/optimizer/DNFFilterOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/DNFFilterOptimizer.java @@ -1,14 +1,15 @@ -package cn.edu.thu.tsfile.optimizer; +package cn.edu.thu.tsfile.qp.optimizer; -import cn.edu.thu.tsfile.common.FilterOperator; -import cn.edu.thu.tsfile.exception.DNFOptimizeException; + +import cn.edu.thu.tsfile.qp.common.FilterOperator; +import cn.edu.thu.tsfile.qp.exception.DNFOptimizeException; import java.util.ArrayList; import java.util.List; -import static cn.edu.thu.tsfile.common.SQLConstant.KW_AND; -import static cn.edu.thu.tsfile.common.SQLConstant.KW_OR; +import static cn.edu.thu.tsfile.qp.common.SQLConstant.KW_AND; +import static cn.edu.thu.tsfile.qp.common.SQLConstant.KW_OR; public class DNFFilterOptimizer implements IFilterOptimizer { diff --git a/src/main/java/cn/edu/thu/tsfile/qp/optimizer/IFilterOptimizer.java b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/IFilterOptimizer.java new file mode 100755 index 0000000..29add72 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/IFilterOptimizer.java @@ -0,0 +1,16 @@ +package cn.edu.thu.tsfile.qp.optimizer; + +import cn.edu.thu.tsfile.qp.common.FilterOperator; +import cn.edu.thu.tsfile.qp.exception.DNFOptimizeException; +import cn.edu.thu.tsfile.qp.exception.MergeFilterException; +import cn.edu.thu.tsfile.qp.exception.RemoveNotException; + +/** + * provide a filter operator, optimize it. + * + * @author kangrong + * + */ +public interface IFilterOptimizer { + FilterOperator optimize(FilterOperator filter) throws RemoveNotException, DNFOptimizeException, MergeFilterException; +} diff --git a/src/main/java/cn/edu/thu/tsfile/optimizer/MergeSingleFilterOptimizer.java b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/MergeSingleFilterOptimizer.java similarity index 96% rename from src/main/java/cn/edu/thu/tsfile/optimizer/MergeSingleFilterOptimizer.java rename to src/main/java/cn/edu/thu/tsfile/qp/optimizer/MergeSingleFilterOptimizer.java index 783527e..0b529a0 100755 --- a/src/main/java/cn/edu/thu/tsfile/optimizer/MergeSingleFilterOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/MergeSingleFilterOptimizer.java @@ -1,8 +1,7 @@ -package cn.edu.thu.tsfile.optimizer; +package cn.edu.thu.tsfile.qp.optimizer; - -import cn.edu.thu.tsfile.exception.MergeFilterException; -import cn.edu.thu.tsfile.common.FilterOperator; +import cn.edu.thu.tsfile.qp.common.FilterOperator; +import cn.edu.thu.tsfile.qp.exception.MergeFilterException; import java.util.ArrayList; import java.util.Collections; diff --git a/src/main/java/cn/edu/thu/tsfile/optimizer/PhysicalOptimizer.java b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/PhysicalOptimizer.java similarity index 98% rename from src/main/java/cn/edu/thu/tsfile/optimizer/PhysicalOptimizer.java rename to src/main/java/cn/edu/thu/tsfile/qp/optimizer/PhysicalOptimizer.java index cac5196..37b65cc 100644 --- a/src/main/java/cn/edu/thu/tsfile/optimizer/PhysicalOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/PhysicalOptimizer.java @@ -1,9 +1,9 @@ -package cn.edu.thu.tsfile.optimizer; +package cn.edu.thu.tsfile.qp.optimizer; -import cn.edu.thu.tsfile.common.*; import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; import cn.edu.thu.tsfile.timeseries.read.metadata.SeriesSchema; import cn.edu.thu.tsfile.timeseries.read.query.QueryEngine; +import cn.edu.thu.tsfile.qp.common.*; import java.io.IOException; import java.util.ArrayList; diff --git a/src/main/java/cn/edu/thu/tsfile/optimizer/RemoveNotOptimizer.java b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/RemoveNotOptimizer.java similarity index 85% rename from src/main/java/cn/edu/thu/tsfile/optimizer/RemoveNotOptimizer.java rename to src/main/java/cn/edu/thu/tsfile/qp/optimizer/RemoveNotOptimizer.java index aeb5f36..8d6a1a9 100755 --- a/src/main/java/cn/edu/thu/tsfile/optimizer/RemoveNotOptimizer.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/optimizer/RemoveNotOptimizer.java @@ -1,19 +1,19 @@ -package cn.edu.thu.tsfile.optimizer; +package cn.edu.thu.tsfile.qp.optimizer; -import cn.edu.thu.tsfile.common.BasicOperator; -import cn.edu.thu.tsfile.exception.BasicOperatorException; -import cn.edu.thu.tsfile.common.FilterOperator; -import cn.edu.thu.tsfile.common.SQLConstant; -import cn.edu.thu.tsfile.exception.RemoveNotException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import cn.edu.thu.tsfile.qp.common.BasicOperator; +import cn.edu.thu.tsfile.qp.common.FilterOperator; +import cn.edu.thu.tsfile.qp.common.SQLConstant; +import cn.edu.thu.tsfile.qp.exception.BasicOperatorException; +import cn.edu.thu.tsfile.qp.exception.RemoveNotException; import java.util.List; -import static cn.edu.thu.tsfile.common.SQLConstant.KW_AND; -import static cn.edu.thu.tsfile.common.SQLConstant.KW_NOT; -import static cn.edu.thu.tsfile.common.SQLConstant.KW_OR; +import static cn.edu.thu.tsfile.qp.common.SQLConstant.KW_AND; +import static cn.edu.thu.tsfile.qp.common.SQLConstant.KW_NOT; +import static cn.edu.thu.tsfile.qp.common.SQLConstant.KW_OR; public class RemoveNotOptimizer implements IFilterOptimizer { diff --git a/src/main/scala/cn/edu/thu/tsfile/Converter.scala b/src/main/scala/cn/edu/thu/tsfile/Converter.scala index 9c55558..e59abb5 100755 --- a/src/main/scala/cn/edu/thu/tsfile/Converter.scala +++ b/src/main/scala/cn/edu/thu/tsfile/Converter.scala @@ -2,10 +2,9 @@ package cn.edu.thu.tsfile import java.util -import cn.edu.thu.tsfile.common.{BasicOperator, FilterOperator, SQLConstant, TSQueryPlan} import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader import cn.edu.thu.tsfile.file.metadata.enums.TSDataType -import cn.edu.thu.tsfile.hadoop.io.HDFSInputStream +import cn.edu.thu.tsfile.io.HDFSInputStream import cn.edu.thu.tsfile.timeseries.read.metadata.SeriesSchema import cn.edu.thu.tsfile.timeseries.read.query.{QueryConfig, QueryEngine} import cn.edu.thu.tsfile.timeseries.read.readSupport.Field @@ -13,6 +12,8 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileStatus import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ +import cn.edu.thu.tsfile.qp.QueryProcessor +import cn.edu.thu.tsfile.qp.common.{BasicOperator, FilterOperator, SQLConstant, TSQueryPlan} import scala.collection.JavaConversions._ import scala.collection.mutable diff --git a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala index badad8e..77d04d8 100755 --- a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala +++ b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala @@ -4,10 +4,9 @@ import java.io._ import java.net.URI import java.util -import cn.edu.thu.tsfile.common.SQLConstant import cn.edu.thu.tsfile.common.constant.QueryConstant -import cn.edu.thu.tsfile.hadoop.io.HDFSInputStream import cn.edu.thu.tsfile.DefaultSource.SerializableConfiguration +import cn.edu.thu.tsfile.io.HDFSInputStream import cn.edu.thu.tsfile.timeseries.read.query.QueryDataSet import cn.edu.thu.tsfile.timeseries.read.readSupport.{Field, RowRecord} import com.esotericsoftware.kryo.io.{Input, Output} @@ -23,7 +22,9 @@ import org.apache.spark.sql.catalyst.expressions.GenericRow import org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriterFactory, PartitionedFile} import org.apache.spark.sql.sources.{DataSourceRegister, Filter} import org.apache.spark.sql.types.{StructField, StructType} -import org.slf4j.{Logger, LoggerFactory} +import org.slf4j.LoggerFactory +import cn.edu.thu.tsfile.qp.Executor +import cn.edu.thu.tsfile.qp.common.SQLConstant import scala.collection.JavaConversions._ import scala.collection.mutable @@ -36,6 +37,8 @@ import scala.collection.mutable */ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { + private val log = LoggerFactory.getLogger(classOf[DefaultSource]) + class TSFileDataSourceException(message: String, cause: Throwable) extends Exception(message, cause){ def this(message: String) = this(message, null) @@ -54,7 +57,7 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { val conf = spark.sparkContext.hadoopConfiguration //check if the path is given - options.getOrElse(DefaultSource.path, throw new TSFileDataSourceException(s"${DefaultSource.path} must be specified for tsfile DataSource")) + options.getOrElse(DefaultSource.path, throw new TSFileDataSourceException(s"${DefaultSource.path} must be specified for cn.edu.thu.tsfile DataSource")) //get union series in TsFile val tsfileSchema = Converter.getUnionSeries(files, conf) @@ -81,16 +84,17 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) (file: PartitionedFile) => { + val log = LoggerFactory.getLogger(classOf[DefaultSource]) + + log.info(file.toString()) val conf = broadcastedConf.value.value val in = new HDFSInputStream(new Path(new URI(file.filePath)), conf) - var taskInfo: String = "" Option(TaskContext.get()).foreach { taskContext => { taskContext.addTaskCompletionListener { _ => in.close() } - taskInfo += "task Id: " + taskContext.taskAttemptId() + " partition Id: " + taskContext.partitionId()} + log.info("task Id: " + taskContext.taskAttemptId() + " partition Id: " + taskContext.partitionId())} } - DefaultSource.logger.debug("taskInfo: {}", taskInfo) val parameters = new util.HashMap[java.lang.String, java.lang.Long]() parameters.put(QueryConstant.PARTITION_START_OFFSET, file.start.asInstanceOf[java.lang.Long]) @@ -99,7 +103,7 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { //convert tsfilequery to QueryConfigs val queryConfigs = Converter.toQueryConfigs(in, requiredSchema, filters, file.start.asInstanceOf[java.lang.Long], (file.start + file.length).asInstanceOf[java.lang.Long]) - //use QueryConfigs to query tsfile + //use QueryConfigs to query cn.edu.thu.tsfile val dataSets = Executor.query(in, queryConfigs.toList, parameters) case class Record(record: RowRecord, index: Int) @@ -198,7 +202,6 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { private[tsfile] object DefaultSource { - val logger: Logger = LoggerFactory.getLogger(getClass) val path = "path" class SerializableConfiguration(@transient var value: Configuration) extends Serializable with KryoSerializable{ diff --git a/src/test/java/cn/edu/thu/tsfile/QueryPlanTest.java b/src/test/java/cn/edu/thu/tsfile/QueryPlanTest.java deleted file mode 100644 index f2c34d4..0000000 --- a/src/test/java/cn/edu/thu/tsfile/QueryPlanTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package cn.edu.thu.tsfile; - - -import cn.edu.thu.tsfile.common.BasicOperator; -import cn.edu.thu.tsfile.common.TSQueryPlan; -import cn.edu.thu.tsfile.common.FilterOperator; -import cn.edu.thu.tsfile.common.SQLConstant; -import cn.edu.thu.tsfile.exception.QueryProcessorException; -import cn.edu.thu.tsfile.timeseries.read.LocalFileInput; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - - -/** - * @author QJL - */ -public class QueryPlanTest { - - private String tsfilePath = "src/test/resources/test.tsfile"; - - @Before - public void before() throws Exception { - new CreateTSFile().createTSFile1(tsfilePath); - } - - @Test - public void testQp() throws IOException, QueryProcessorException { - LocalFileInput in = new LocalFileInput(tsfilePath); - FilterOperator filterOperator = new FilterOperator(SQLConstant.KW_AND); - filterOperator.addChildOPerator(new BasicOperator(SQLConstant.GREATERTHAN, "time", "50")); - filterOperator.addChildOPerator(new BasicOperator(SQLConstant.GREATERTHAN, "s1", "80")); - - ArrayList paths = new ArrayList<>(); - paths.add("s1"); - paths.add("time"); - - List queryPlans = new QueryProcessor().generatePlans(filterOperator, paths, in, Long.valueOf("0"), Long.valueOf("749")); - - ArrayList expectedPaths1 = new ArrayList<>(); - expectedPaths1.add("root.car.d2.s1"); - FilterOperator expectedTimeFilterOperator1 = new BasicOperator(SQLConstant.GREATERTHAN, "time", "50"); - FilterOperator expectedValueFilterOperator1 = new BasicOperator(SQLConstant.GREATERTHAN, "s1", "80"); - TSQueryPlan expectedQueryPlan1 = new TSQueryPlan(expectedPaths1, expectedTimeFilterOperator1, expectedValueFilterOperator1); - - ArrayList expectedPaths2 = new ArrayList<>(); - expectedPaths2.add("root.car.d1.s1"); - FilterOperator expectedTimeFilterOperator2 = new BasicOperator(SQLConstant.GREATERTHAN, "time", "50"); - FilterOperator expectedValueFilterOperator2 = new BasicOperator(SQLConstant.GREATERTHAN, "s1", "80"); - TSQueryPlan expectedQueryPlan2 = new TSQueryPlan(expectedPaths2, expectedTimeFilterOperator2, expectedValueFilterOperator2); - - Assert.assertEquals(2, queryPlans.size()); - Assert.assertEquals(expectedQueryPlan1.toString(), queryPlans.get(0).toString()); - Assert.assertEquals(expectedQueryPlan2.toString(), queryPlans.get(1).toString()); - } -} diff --git a/src/test/java/cn/edu/thu/tsfile/hadoop/io/HDFSInputOutputStreamTest.java b/src/test/java/cn/edu/thu/tsfile/hadoop/io/HDFSInputOutputStreamTest.java deleted file mode 100644 index d30bb36..0000000 --- a/src/test/java/cn/edu/thu/tsfile/hadoop/io/HDFSInputOutputStreamTest.java +++ /dev/null @@ -1,67 +0,0 @@ -package cn.edu.thu.tsfile.hadoop.io; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - - -public class HDFSInputOutputStreamTest { - - private HDFSInputStream hdfsInputStream = null; - private HDFSOutputStream hdfsOutputStream = null; - private int lenOfBytes = 50; - private byte b = 10; - private byte[] bs = new byte[lenOfBytes]; - private byte[] rbs = new byte[lenOfBytes]; - private String filename = "testinputandoutputstream.file"; - private Path path; - private FileSystem fileSystem; - - @Before - public void setUp() throws Exception { - hdfsOutputStream = new HDFSOutputStream(filename, new Configuration(), true); - path = new Path(filename); - hdfsInputStream = new HDFSInputStream(filename); - } - - @After - public void tearDown() throws Exception { - if (fileSystem.exists(path)) { - fileSystem.delete(path, true); - } - - } - - @Test - public void test() throws Exception { - // write one byte - hdfsOutputStream.write(b); - assertEquals(1, hdfsOutputStream.getPos()); - hdfsOutputStream.close(); - fileSystem = FileSystem.get(new Configuration()); - assertEquals(true, fileSystem.exists(path)); - fileSystem.delete(path, true); - // write bytes - hdfsOutputStream = new HDFSOutputStream(filename, new Configuration(), true); - hdfsOutputStream.write(bs); - assertEquals(bs.length, hdfsOutputStream.getPos()); - hdfsOutputStream.close(); - // read bytes using hdfs inputstream - hdfsInputStream = new HDFSInputStream(filename); - assertEquals(0, hdfsInputStream.getPos()); - assertEquals(lenOfBytes, hdfsInputStream.length()); - hdfsInputStream.seek(10); - assertEquals(10, hdfsInputStream.getPos()); - hdfsInputStream.seek(0); - hdfsInputStream.read(rbs, 0, rbs.length); - assertEquals(lenOfBytes, hdfsInputStream.getPos()); - assertArrayEquals(bs, rbs); - } - -} \ No newline at end of file diff --git a/src/test/resources/test.tsfile b/src/test/resources/test.tsfile deleted file mode 100644 index f733bfc64107f6d498140881eaf3c800bb9732cf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1233 zcmai!KTE?v7{)J`v%SW_BHWQe2N#Qil>QaPp`}7CMZ`fIT*N;u2!h%K(M`lb`~+G$ zs`vq1+#DSgiiqImAc%{D_yK$`O^!>07@hLMB2;hua&PI~l;kLptePO}DJVr}q-v>Z~r^XzA-hjJ8Qj z@9v&3S0k#E&7Y#!4Q>@yk!}Q*s41QUVZB9(PL>ZGhZ?I%2P#})ESlad7v|(|rw>)L zX_CrAK95qA 5 or s1 > 10)") Assert.assertEquals(0, newDf.count()) } test("testMultiFilesNoneExistDelta_object") { - val df = spark.read.format("cn.edu.thu.tsfile").load(tsfileFolder) + val df = spark.read.tsfile(tsfileFolder) df.createOrReplaceTempView("tsfile_table") val newDf = spark.sql("select * from tsfile_table where delta_object = 'd4'") Assert.assertEquals(0, newDf.count()) } test("testMultiFilesWithFilterOr") { - val df = spark.read.format("cn.edu.thu.tsfile").load(tsfileFolder) + val df = spark.read.tsfile(tsfileFolder) df.createOrReplaceTempView("tsfile_table") val newDf = spark.sql("select * from tsfile_table where s1 < 2 or s2 > 60") Assert.assertEquals(4, newDf.count()) @@ -113,7 +114,7 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { } test("testQuerySchema") { - val df = spark.read.tsfile(tsfilePath1) + val df = spark.read.format("cn.edu.thu.tsfile").load(tsfilePath1) val expected = StructType(Seq( StructField(SQLConstant.RESERVED_TIME, LongType, nullable = true), @@ -126,4 +127,13 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { Assert.assertEquals(expected, df.schema) } + test("testQuerySchema1") { + val df = spark.read.format("cn.edu.thu.tsfile").load("/testdata/test1.tsfile") + df.createOrReplaceTempView("tsfile") + + println(spark.sql("select * from tsfile").schema) + + spark.sql("select * from tsfile").show() + } + } \ No newline at end of file From 5935f4bc95d1cb47c137140fffea5527db4466b9 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Tue, 13 Jun 2017 21:20:00 +0800 Subject: [PATCH 06/15] remove unused test --- .../scala/cn/edu/thu/tsfile/ConverterTest.scala | 2 +- .../scala/cn/edu/thu/tsfile/TSFileSuit.scala | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala b/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala index d20b979..68aff6d 100644 --- a/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala +++ b/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala @@ -22,7 +22,7 @@ import scala.collection.mutable.ListBuffer */ class ConverterTest extends FunSuite with BeforeAndAfterAll { - private val tsfilePath: String = "src/test/resources/test.cn.edu.thu.tsfile" + private val tsfilePath: String = "src/test/resources/test.tsfile" override protected def beforeAll(): Unit = { super.beforeAll() diff --git a/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala index 357894b..9c7469e 100644 --- a/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala +++ b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala @@ -127,13 +127,13 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { Assert.assertEquals(expected, df.schema) } - test("testQuerySchema1") { - val df = spark.read.format("cn.edu.thu.tsfile").load("/testdata/test1.tsfile") - df.createOrReplaceTempView("tsfile") - - println(spark.sql("select * from tsfile").schema) - - spark.sql("select * from tsfile").show() - } +// test("testQuerySchema1") { +// val df = spark.read.format("cn.edu.thu.tsfile").load("/testdata/test1.tsfile") +// df.createOrReplaceTempView("tsfile") +// +// println(spark.sql("select * from tsfile").schema) +// +// spark.sql("select * from tsfile").show() +// } } \ No newline at end of file From 857b98b42ac67bbd8f4b87ed2c3937a378b18658 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Wed, 14 Jun 2017 11:25:46 +0800 Subject: [PATCH 07/15] update readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f395460..da78ce1 100644 --- a/README.md +++ b/README.md @@ -104,13 +104,13 @@ The SparkSQL Table Structure is as follow: ```scala // import this library and Spark - import cn.edu.thu.cn.edu.thu.tsfile._ + import cn.edu.thu.tsfile._ import org.apache.spark.sql.SparkSession val spark = SparkSession.builder().master("local").getOrCreate() //read data in TsFile and create a table - val df = spark.read.cn.edu.thu.tsfile("test.ts") + val df = spark.read.tsfile("test.ts") df.createOrReplaceTempView("TsFile_table") //query with filter @@ -123,7 +123,7 @@ The SparkSQL Table Structure is as follow: * **Example 2** ```scala - import cn.edu.thu.cn.edu.thu.tsfile._ + import cn.edu.thu.tsfile._ import org.apache.spark.sql.SparkSession val spark = SparkSession.builder().master("local").getOrCreate() @@ -139,7 +139,7 @@ The SparkSQL Table Structure is as follow: * **Example 3** ```scala - import cn.edu.thu.cn.edu.thu.tsfile._ + import cn.edu.thu.tsfile._ import org.apache.spark.sql.SparkSession val spark = SparkSession.builder().master("local").getOrCreate() From f4d70bd91ac3896735e68767a25eaed47d2fec02 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Wed, 14 Jun 2017 11:28:04 +0800 Subject: [PATCH 08/15] update readme --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index da78ce1..430b3ee 100644 --- a/README.md +++ b/README.md @@ -156,16 +156,17 @@ The SparkSQL Table Structure is as follow: 可以将项目打包在 `spark-shell`中使用。 ``` -mvn package -DskipTests +mvn clean scala:compile compile package -DskipTests +``` 包所在位置： -/cn.edu.thu.tsfile-kmx-spark-connector/target/cn.edu.thu.tsfile-1.0-SNAPSHOT-jar-with-dependencies.jar -``` +target/tsfile-spark-connector-0.1.0.jar + ``` -$ bin/spark-shell --jars cn.edu.thu.tsfile-spark-0.1.0-jar-with-dependencies.jar +$ bin/spark-shell --jars target/tsfile-spark-connector-0.1.0.jar,tsfile-0.1.0.jar -scala> sql("CREATE TEMPORARY TABLE TsFile_table USING cn.edu.thu.cn.edu.thu.tsfile.spark OPTIONS (path \"hdfs://localhost:9000/test.ts\")") +scala> sql("CREATE TEMPORARY TABLE TsFile_table USING cn.edu.thu.cn.edu.thu.tsfile OPTIONS (path \"hdfs://localhost:9000/test1.tsfile\")") -scala> sql("select * from TsFile_table where sensor_1 > 1.2").show() +scala> sql("select * from TsFile_table").show() ``` \ No newline at end of file From cf2141d57b243ebdb97d7a25cb9f5b828b4ad785 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Wed, 14 Jun 2017 21:07:24 +0800 Subject: [PATCH 09/15] fix hadoop serialization bug --- .../cn/edu/thu/tsfile/DefaultSource.scala | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala index 77d04d8..970e91b 100755 --- a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala +++ b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala @@ -9,8 +9,6 @@ import cn.edu.thu.tsfile.DefaultSource.SerializableConfiguration import cn.edu.thu.tsfile.io.HDFSInputStream import cn.edu.thu.tsfile.timeseries.read.query.QueryDataSet import cn.edu.thu.tsfile.timeseries.read.readSupport.{Field, RowRecord} -import com.esotericsoftware.kryo.io.{Input, Output} -import com.esotericsoftware.kryo.{Kryo, KryoSerializable} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.hadoop.mapreduce.Job @@ -25,6 +23,7 @@ import org.apache.spark.sql.types.{StructField, StructType} import org.slf4j.LoggerFactory import cn.edu.thu.tsfile.qp.Executor import cn.edu.thu.tsfile.qp.common.SQLConstant +import java.io.{ObjectInputStream, ObjectOutputStream} import scala.collection.JavaConversions._ import scala.collection.mutable @@ -84,11 +83,14 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) (file: PartitionedFile) => { + println(file.toString()) val log = LoggerFactory.getLogger(classOf[DefaultSource]) log.info(file.toString()) val conf = broadcastedConf.value.value + if(conf == null) + println("@+++<<<: conf is null!!!!!!!") val in = new HDFSInputStream(new Path(new URI(file.filePath)), conf) Option(TaskContext.get()).foreach { taskContext => { @@ -204,16 +206,15 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { private[tsfile] object DefaultSource { val path = "path" - class SerializableConfiguration(@transient var value: Configuration) extends Serializable with KryoSerializable{ - override def write(kryo: Kryo, output: Output): Unit = { - val dos = new DataOutputStream(output) - value.write(dos) - dos.flush() + class SerializableConfiguration(@transient var value: Configuration) extends Serializable { + private def writeObject(out: ObjectOutputStream): Unit = { + out.defaultWriteObject() + value.write(out) } - override def read(kryo: Kryo, input: Input): Unit = { + private def readObject(in: ObjectInputStream): Unit = { value = new Configuration(false) - value.readFields(new DataInputStream(input)) + value.readFields(in) } } From 2a9bb515e1eb00025694fb8aac441d57a7fe1daa Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Wed, 14 Jun 2017 21:14:49 +0800 Subject: [PATCH 10/15] remove print --- src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala index 970e91b..5009827 100755 --- a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala +++ b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala @@ -83,14 +83,11 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) (file: PartitionedFile) => { - println(file.toString()) val log = LoggerFactory.getLogger(classOf[DefaultSource]) log.info(file.toString()) val conf = broadcastedConf.value.value - if(conf == null) - println("@+++<<<: conf is null!!!!!!!") val in = new HDFSInputStream(new Path(new URI(file.filePath)), conf) Option(TaskContext.get()).foreach { taskContext => { From 29d146aaccdf7b0a305c3150da5aa5cea18e927d Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Wed, 14 Jun 2017 21:34:07 +0800 Subject: [PATCH 11/15] update readme --- README.md | 10 +++++----- .../scala/cn/edu/thu/tsfile/DefaultSource.scala | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 430b3ee..7a390e9 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ The SparkSQL Table Structure is as follow: val spark = SparkSession.builder().master("local").getOrCreate() val df = spark.read - .format("cn.edu.thu.cn.edu.thu.tsfile.spark") + .format("cn.edu.thu.tsfile") .load("test.ts") @@ -145,7 +145,7 @@ The SparkSQL Table Structure is as follow: val spark = SparkSession.builder().master("local").getOrCreate() //create a table in SparkSQL and build relation with a TsFile - spark.sql("create temporary view TsFile using cn.edu.thu.cn.edu.thu.tsfile.spark options(path = \"test.ts\")") + spark.sql("create temporary view TsFile using cn.edu.thu.tsfile options(path = \"test.ts\")") spark.sql("select * from TsFile where sensor_1 > 1.2").show() @@ -156,7 +156,7 @@ The SparkSQL Table Structure is as follow: 可以将项目打包在 `spark-shell`中使用。 ``` -mvn clean scala:compile compile package -DskipTests +mvn clean scala:compile compile package ``` 包所在位置： @@ -164,9 +164,9 @@ target/tsfile-spark-connector-0.1.0.jar ``` -$ bin/spark-shell --jars target/tsfile-spark-connector-0.1.0.jar,tsfile-0.1.0.jar +$ bin/spark-shell --jars tsfile-spark-connector-0.1.0.jar,tsfile-0.1.0.jar -scala> sql("CREATE TEMPORARY TABLE TsFile_table USING cn.edu.thu.cn.edu.thu.tsfile OPTIONS (path \"hdfs://localhost:9000/test1.tsfile\")") +scala> sql("CREATE TEMPORARY TABLE TsFile_table USING cn.edu.thu.tsfile OPTIONS (path \"hdfs://localhost:9000/test1.tsfile\")") scala> sql("select * from TsFile_table").show() ``` \ No newline at end of file diff --git a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala index 5009827..33aa3c6 100755 --- a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala +++ b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala @@ -39,7 +39,7 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { private val log = LoggerFactory.getLogger(classOf[DefaultSource]) class TSFileDataSourceException(message: String, cause: Throwable) - extends Exception(message, cause){ + extends Exception(message, cause) { def this(message: String) = this(message, null) } @@ -92,7 +92,8 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { Option(TaskContext.get()).foreach { taskContext => { taskContext.addTaskCompletionListener { _ => in.close() } - log.info("task Id: " + taskContext.taskAttemptId() + " partition Id: " + taskContext.partitionId())} + log.info("task Id: " + taskContext.taskAttemptId() + " partition Id: " + taskContext.partitionId()) + } } val parameters = new util.HashMap[java.lang.String, java.lang.Long]() @@ -109,9 +110,9 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { implicit object RowRecordOrdering extends Ordering[Record] { override def compare(r1: Record, r2: Record): Int = { - if(r1.record.timestamp == r2.record.timestamp) { + if (r1.record.timestamp == r2.record.timestamp) { r1.record.getFields.get(0).deltaObjectId.compareTo(r2.record.getFields.get(0).deltaObjectId) - } else if(r1.record.timestamp < r2.record.timestamp){ + } else if (r1.record.timestamp < r2.record.timestamp) { 1 } else { -1 @@ -123,9 +124,9 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { //init priorityQueue with first record of each dataSet var queryDataSet: QueryDataSet = null - for(i <- 0 until dataSets.size()) { + for (i <- 0 until dataSets.size()) { queryDataSet = dataSets(i) - if(queryDataSet.hasNextRecord) { + if (queryDataSet.hasNextRecord) { val rowRecord = queryDataSet.getNextRecord priorityQueue.enqueue(Record(rowRecord, i)) } From 4106bfcd65f1e716b45ef4b070ca964d5acb3503 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Fri, 16 Jun 2017 13:35:11 +0800 Subject: [PATCH 12/15] update readme --- README.md | 18 ++++---- .../scala/cn/edu/thu/tsfile/TSFileSuit.scala | 44 ++++++++++++++++++- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7a390e9..469f99c 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,21 @@ -# cn.edu.thu.tsfile-spark-connector +# tsfile-spark-connector + +Used to read tsfile in spark. 将一个或多个TsFile展示成SparkSQL中的一张表。允许指定单个目录，或使用通配符匹配多个目录。如果是多个TsFile，schema将保留各个TsFile中sensor的并集。 -## 示例 +## Example -src/test/scala/cn.edu.thu.cn.edu.thu.tsfile.spark.TSFileSuit +src/test/scala/cn.edu.thu.tsfile.spark.TSFileSuit ## 路径指定方式 -basefolder/key=1/file1.cn.edu.thu.tsfile +basefolder/key=1/file1.tsfile -basefolder/key=2/file2.cn.edu.thu.tsfile +basefolder/key=2/file2.tsfile 指定basefolder为path，会在表中多加一列key，值为1或2。 如： @@ -23,11 +25,11 @@ path=basefolder 如果使用通配符指定，将不会当做partiton 如： -path=basefolder/\*/\*.cn.edu.thu.tsfile +path=basefolder/\*/\*.tsfile -basefolder/file1.cn.edu.thu.tsfile -basefolder/file2.cn.edu.thu.tsfile +basefolder/file1.tsfile +basefolder/file2.tsfile 指定basefolder会将多个tsfile的schema合并，保留sensor的并集 diff --git a/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala index 9c7469e..ecba811 100644 --- a/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala +++ b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala @@ -3,7 +3,7 @@ package cn.edu.thu.tsfile import java.io.File import cn.edu.thu.tsfile.io.CreateTSFile -import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.types._ import org.junit.Assert import org.scalatest.{BeforeAndAfterAll, FunSuite} @@ -12,6 +12,16 @@ import cn.edu.thu.tsfile.qp.common.SQLConstant /** * @author QJL */ + +class Seq extends Serializable { + var i = 0 + + def getVal: Int = { + i = i + 1 + i + } +} + class TSFileSuit extends FunSuite with BeforeAndAfterAll { private val resourcesFolder = "src/test/resources" @@ -32,7 +42,7 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { new CreateTSFile().createTSFile2(tsfilePath2) spark = SparkSession .builder() - .config("spark.master", "local") + .config("spark.master", "spark://192.168.130.15:7077") .appName("TSFile test") .getOrCreate() } @@ -45,6 +55,36 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { } } + test("afa") { + val rdd = spark.sparkContext.parallelize(Seq(Row("one"), Row("two"))) + + val schema = StructType(Array(StructField("name", StringType))) + + val df = spark.createDataFrame(rdd, schema) + + df.show() + + spark.udf.register("func", (name: String) => name.toUpperCase) + + import org.apache.spark.sql.functions.expr + + val newDf = df.withColumn("upperName", expr("func(name)")) + +// newDf.show() +// +// val seq = new Seq +// +// spark.udf.register("seq", () => seq.getVal) +// +// val seqDf = df.withColumn("id", expr("seq()")) +// +// seqDf.show() +// +// df.createOrReplaceTempView("df") +// +// spark.sql("select *, seq() as sql_id from df").show() + } + test("tsfile/qp") { val df = spark.read.tsfile(tsfileFolder) df.createOrReplaceTempView("tsfile_table") From 72846377d5b202ae72f9c857de79f23a122f47fe Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Fri, 23 Jun 2017 21:49:09 +0800 Subject: [PATCH 13/15] init writer --- README.md | 6 +- .../cn/edu/thu/tsfile/io/CreateTSFile.java | 4 ++ .../cn/edu/thu/tsfile/DefaultSource.scala | 8 +-- .../edu/thu/tsfile/TsFileOutputWriter.scala | 26 ++++++++ .../edu/thu/tsfile/TsFileWriterFactory.scala | 23 +++++++ .../scala/cn/edu/thu/tsfile/package.scala | 13 +++- src/test/resources/tsfile/test1.tsfile | Bin 1233 -> 0 bytes .../scala/cn/edu/thu/tsfile/TSFileSuit.scala | 61 ++++++------------ 8 files changed, 88 insertions(+), 53 deletions(-) create mode 100644 src/main/scala/cn/edu/thu/tsfile/TsFileOutputWriter.scala create mode 100644 src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala delete mode 100644 src/test/resources/tsfile/test1.tsfile diff --git a/README.md b/README.md index 469f99c..9b62e7c 100644 --- a/README.md +++ b/README.md @@ -131,10 +131,8 @@ The SparkSQL Table Structure is as follow: val spark = SparkSession.builder().master("local").getOrCreate() val df = spark.read .format("cn.edu.thu.tsfile") - .load("test.ts") - - - df.filter("sensor_1 > 1.2").show() + .load("test.tsfile") + df.filter("time < 10").show() ``` diff --git a/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java b/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java index caa57be..c253899 100755 --- a/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java +++ b/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java @@ -28,6 +28,10 @@ public void createTSFile1(String tsfilePath) throws Exception { TSRandomAccessFileWriter output = new RandomAccessOutputStream(new File(tsfilePath)); TsFile tsFile = new TsFile(output, jsonSchema); +// for( int i = 1; i < 30000000; i++) { +// tsFile.writeLine("root.car.d1," + i + ", s1, " + i + ", s2, 10, s3, 100"); +// } + tsFile.writeLine("root.car.d1,1, s1, 1, s2, 10, s3, 100.1, s4, 0.1"); tsFile.writeLine("root.car.d1,2, s1, 2, s2, 20, s3, 200.2, s4, 0.2"); tsFile.writeLine("root.car.d1,3, s1, 3, s2, 30, s3, 200.3, s4, 0.3"); diff --git a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala index 33aa3c6..832a0c1 100755 --- a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala +++ b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala @@ -36,8 +36,6 @@ import scala.collection.mutable */ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { - private val log = LoggerFactory.getLogger(classOf[DefaultSource]) - class TSFileDataSourceException(message: String, cause: Throwable) extends Exception(message, cause) { def this(message: String) = this(message, null) @@ -84,7 +82,6 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { (file: PartitionedFile) => { val log = LoggerFactory.getLogger(classOf[DefaultSource]) - log.info(file.toString()) val conf = broadcastedConf.value.value @@ -193,9 +190,10 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { override def shortName(): String = "tsfile" override def prepareWrite(sparkSession: SparkSession, - job: Job, options: Map[String, String], + job: Job, + options: Map[String, String], dataSchema: StructType): OutputWriterFactory = { - null + new TsFileWriterFactory() } } diff --git a/src/main/scala/cn/edu/thu/tsfile/TsFileOutputWriter.scala b/src/main/scala/cn/edu/thu/tsfile/TsFileOutputWriter.scala new file mode 100644 index 0000000..dbe98dc --- /dev/null +++ b/src/main/scala/cn/edu/thu/tsfile/TsFileOutputWriter.scala @@ -0,0 +1,26 @@ +package cn.edu.thu.tsfile + +import cn.edu.thu.tsfile.io.HDFSOutputStream +import cn.edu.thu.tsfile.timeseries.FileFormat.TsFile +import org.apache.hadoop.mapreduce.TaskAttemptContext +import org.apache.spark.sql.Row +import org.apache.spark.sql.execution.datasources.OutputWriter +import org.json.JSONObject + + +private[tsfile] class TsFileOutputWriter(path: String, context: TaskAttemptContext) extends OutputWriter{ + + private val tsfile = { + val conf = context.getConfiguration + val hdfsOutput = new HDFSOutputStream(path, conf, true) + new TsFile(hdfsOutput, new JSONObject("")) + } + + override def write(row: Row): Unit = { + tsfile.writeLine("") + } + + override def close(): Unit = { + tsfile.close() + } +} diff --git a/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala b/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala new file mode 100644 index 0000000..b64ef84 --- /dev/null +++ b/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala @@ -0,0 +1,23 @@ +package cn.edu.thu.tsfile + +import org.apache.hadoop.mapreduce.TaskAttemptContext +import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory} +import org.apache.spark.sql.types.StructType + +/** + * @author qiaojialin + */ +private[tsfile] class TsFileWriterFactory() extends OutputWriterFactory{ + + override def newInstance( + path: String, + bucketId: Option[Int], + dataSchema: StructType, + context: TaskAttemptContext): OutputWriter = { + println("---newInstance---") + + println(dataSchema) + + new TsFileOutputWriter(path, context) + } +} diff --git a/src/main/scala/cn/edu/thu/tsfile/package.scala b/src/main/scala/cn/edu/thu/tsfile/package.scala index f1e6ba7..b01d260 100755 --- a/src/main/scala/cn/edu/thu/tsfile/package.scala +++ b/src/main/scala/cn/edu/thu/tsfile/package.scala @@ -1,13 +1,20 @@ package cn.edu.thu -import org.apache.spark.sql.{DataFrame, DataFrameReader} +import org.apache.spark.sql.{DataFrame, DataFrameReader, DataFrameWriter} package object tsfile { /** - * add a method to DataFrameReader + * add a method 'tsfile' to DataFrameReader to read tsfile */ - implicit class TSFileDataFrameReader(reader: DataFrameReader) { + implicit class TsFileDataFrameReader(reader: DataFrameReader) { def tsfile: String => DataFrame = reader.format("cn.edu.thu.tsfile").load } + + /** + * add a method 'tsfile' to DataFrameWriter to write tsfile + */ + implicit class TsFileDataFrameWriter[T](writer: DataFrameWriter[T]) { + def tsfile: String => Unit = writer.format("cn.edu.thu.tsfile").save + } } diff --git a/src/test/resources/tsfile/test1.tsfile b/src/test/resources/tsfile/test1.tsfile deleted file mode 100644 index f733bfc64107f6d498140881eaf3c800bb9732cf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1233 zcmai!KTE?v7{)J`v%SW_BHWQe2N#Qil>QaPp`}7CMZ`fIT*N;u2!h%K(M`lb`~+G$ zs`vq1+#DSgiiqImAc%{D_yK$`O^!>07@hLMB2;hua&PI~l;kLptePO}DJVr}q-v>Z~r^XzA-hjJ8Qj z@9v&3S0k#E&7Y#!4Q>@yk!}Q*s41QUVZB9(PL>ZGhZ?I%2P#})ESlad7v|(|rw>)L zX_CrAK95qA name.toUpperCase) - - import org.apache.spark.sql.functions.expr - - val newDf = df.withColumn("upperName", expr("func(name)")) - -// newDf.show() -// -// val seq = new Seq -// -// spark.udf.register("seq", () => seq.getVal) -// -// val seqDf = df.withColumn("id", expr("seq()")) -// -// seqDf.show() -// -// df.createOrReplaceTempView("df") -// -// spark.sql("select *, seq() as sql_id from df").show() + df.write.tsfile(outputPath) } - test("tsfile/qp") { + test("tsfile_qp") { val df = spark.read.tsfile(tsfileFolder) df.createOrReplaceTempView("tsfile_table") val newDf = spark.sql("select s1,s2 from tsfile_table where delta_object = 'root.car.d1' and time <= 10 and (time > 5 or s1 > 10)") From f8d2572e37758e90dbc34dea7652b4c1ce87e708 Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Mon, 26 Jun 2017 17:26:21 +0800 Subject: [PATCH 14/15] local write pass --- README.md | 42 +------ data/test1.tsfile | Bin 0 -> 1233 bytes .../cn/edu/thu/tsfile/io/CreateTSFile.java | 6 +- .../edu/thu/tsfile/io/HDFSOutputStream.java | 4 - .../edu/thu/tsfile/io/TsFileOutputFormat.java | 33 +++++ .../edu/thu/tsfile/io/TsFileRecordWriter.java | 37 ++++++ .../edu/thu/tsfile/qp/common/SQLConstant.java | 23 +--- .../scala/cn/edu/thu/tsfile/Converter.scala | 113 ++++++++++++++++-- .../cn/edu/thu/tsfile/DefaultSource.scala | 8 +- .../edu/thu/tsfile/TsFileOutputWriter.scala | 30 +++-- .../edu/thu/tsfile/TsFileWriterFactory.scala | 10 +- src/test/resources/test.tsfile | Bin 0 -> 1233 bytes src/test/resources/tsfile/test1.tsfile | Bin 0 -> 1233 bytes .../cn/edu/thu/tsfile/ConverterTest.scala | 2 +- .../scala/cn/edu/thu/tsfile/TSFileSuit.scala | 37 ++++-- 15 files changed, 241 insertions(+), 104 deletions(-) create mode 100644 data/test1.tsfile create mode 100644 src/main/java/cn/edu/thu/tsfile/io/TsFileOutputFormat.java create mode 100644 src/main/java/cn/edu/thu/tsfile/io/TsFileRecordWriter.java create mode 100644 src/test/resources/test.tsfile create mode 100644 src/test/resources/tsfile/test1.tsfile diff --git a/README.md b/README.md index 9b62e7c..50d4225 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,16 @@ # tsfile-spark-connector -Used to read tsfile in spark. +Used to read and write(developing) tsfile in spark. 将一个或多个TsFile展示成SparkSQL中的一张表。允许指定单个目录，或使用通配符匹配多个目录。如果是多个TsFile，schema将保留各个TsFile中sensor的并集。 -## Example +## dependency -src/test/scala/cn.edu.thu.tsfile.spark.TSFileSuit +https://github.com/thulab/tsfile.git -## 路径指定方式 - - -basefolder/key=1/file1.tsfile - -basefolder/key=2/file2.tsfile -指定basefolder为path，会在表中多加一列key，值为1或2。 - -如： -path=basefolder - - -如果使用通配符指定，将不会当做partiton - -如： -path=basefolder/\*/\*.tsfile - - -basefolder/file1.tsfile -basefolder/file2.tsfile - -指定basefolder会将多个tsfile的schema合并，保留sensor的并集 - -如： -path=basefolder - - -## 版本需求 +## versions The versions required for Spark and Java are as follow: @@ -47,7 +20,7 @@ The versions required for Spark and Java are as follow: -## 数据类型转化 +## TsFile Type <=> SparkSQL type This library uses the following mapping the data type from TsFile to SparkSQL: @@ -153,15 +126,12 @@ The SparkSQL Table Structure is as follow: ##### spark-shell -可以将项目打包在 `spark-shell`中使用。 +package: ``` mvn clean scala:compile compile package ``` -包所在位置： -target/tsfile-spark-connector-0.1.0.jar - ``` $ bin/spark-shell --jars tsfile-spark-connector-0.1.0.jar,tsfile-0.1.0.jar diff --git a/data/test1.tsfile b/data/test1.tsfile new file mode 100644 index 0000000000000000000000000000000000000000..9522642be492d9a4e6259d48abe5da76e3b0e312 GIT binary patch literal 1233 zcmai!PfHvz7{)W1I2*@WANx)}rqBB{pe$HhZkCW;&+?t|k+!&U4puPiHwdI)0mfmZJwrCp%u3QoCf_PTV za!QudhiY>(V9oPdjg+{g&!ciQs`<&OpL_pq4^#mNTuztf35+5D{V}L6o@UAww^a`S za})6cP&(+j?!vv=fZ42=HTP7Em?q_9l?Dqo_Cp!8M9Ub&ct#BJ+#nYM|_f zFY~0RoRU2Zd^D^>^W8~_{78gUihsIQwYUQDHEx28M^;7}iLrx>xOLo_T2zJTm#al5 zmWV|nKtM)_RM^c3@lbbXr&nHI?UpyH@;8(DQ466#@BoVx11h=2z%-{}@X3S7lXgXZ zgK1I3Al-w8!C6jcJhZMu8&4SxgSh7mgRgiHl=;~bu}G9)VE0*)K}BBoF?gk*(X~bE GPwYPx=AaS) literal 0 HcmV?d00001 diff --git a/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java b/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java index c253899..a543bd9 100755 --- a/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java +++ b/src/main/java/cn/edu/thu/tsfile/io/CreateTSFile.java @@ -28,11 +28,7 @@ public void createTSFile1(String tsfilePath) throws Exception { TSRandomAccessFileWriter output = new RandomAccessOutputStream(new File(tsfilePath)); TsFile tsFile = new TsFile(output, jsonSchema); -// for( int i = 1; i < 30000000; i++) { -// tsFile.writeLine("root.car.d1," + i + ", s1, " + i + ", s2, 10, s3, 100"); -// } - - tsFile.writeLine("root.car.d1,1, s1, 1, s2, 10, s3, 100.1, s4, 0.1"); + tsFile.writeLine("root.car.d1,1, s1, 1, s2, 10, s3, 100.1"); tsFile.writeLine("root.car.d1,2, s1, 2, s2, 20, s3, 200.2, s4, 0.2"); tsFile.writeLine("root.car.d1,3, s1, 3, s2, 30, s3, 200.3, s4, 0.3"); tsFile.writeLine("root.car.d1,4, s1, 4, s2, 40, s3, 200.4, s4, 0.4"); diff --git a/src/main/java/cn/edu/thu/tsfile/io/HDFSOutputStream.java b/src/main/java/cn/edu/thu/tsfile/io/HDFSOutputStream.java index 272b5f9..729702c 100644 --- a/src/main/java/cn/edu/thu/tsfile/io/HDFSOutputStream.java +++ b/src/main/java/cn/edu/thu/tsfile/io/HDFSOutputStream.java @@ -5,8 +5,6 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.OutputStream; @@ -20,8 +18,6 @@ */ public class HDFSOutputStream implements TSRandomAccessFileWriter { - private static final Logger LOGGER = LoggerFactory.getLogger(HDFSOutputStream.class); - private FSDataOutputStream fsDataOutputStream; public HDFSOutputStream(String filePath, boolean overwriter) throws IOException { diff --git a/src/main/java/cn/edu/thu/tsfile/io/TsFileOutputFormat.java b/src/main/java/cn/edu/thu/tsfile/io/TsFileOutputFormat.java new file mode 100644 index 0000000..a150712 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfile/io/TsFileOutputFormat.java @@ -0,0 +1,33 @@ +package cn.edu.thu.tsfile.io; + +import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException; +import cn.edu.thu.tsfile.timeseries.write.record.TSRecord; +import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import java.io.IOException; + +public class TsFileOutputFormat extends FileOutputFormat { + + private FileSchema fileSchema; + + public TsFileOutputFormat(FileSchema fileSchema) { + this.fileSchema = fileSchema; + } + + @Override + public RecordWriter getRecordWriter(TaskAttemptContext job) + throws IOException, InterruptedException { + Path path = getDefaultWorkFile(job, ""); + try { + return new TsFileRecordWriter(job, path, fileSchema); + } catch (WriteProcessException e) { + e.printStackTrace(); + throw new InterruptedException("construct TsFileRecordWriter failed"); + } + } + +} diff --git a/src/main/java/cn/edu/thu/tsfile/io/TsFileRecordWriter.java b/src/main/java/cn/edu/thu/tsfile/io/TsFileRecordWriter.java new file mode 100644 index 0000000..4d9dc33 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfile/io/TsFileRecordWriter.java @@ -0,0 +1,37 @@ +package cn.edu.thu.tsfile.io; + +import cn.edu.thu.tsfile.timeseries.FileFormat.TsFile; +import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException; +import cn.edu.thu.tsfile.timeseries.write.record.TSRecord; +import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; + +public class TsFileRecordWriter extends RecordWriter { + + private TsFile tsFile = null; + + public TsFileRecordWriter(TaskAttemptContext job, Path file, FileSchema fileSchema) throws IOException, WriteProcessException { + HDFSOutputStream hdfsOutputStream = new HDFSOutputStream(file.toString(), job.getConfiguration(), false); + tsFile = new TsFile(hdfsOutputStream, fileSchema); + } + + @Override + public void close(TaskAttemptContext context) throws IOException { + tsFile.close(); + } + + @Override + public synchronized void write(NullWritable arg0, TSRecord tsRecord) throws IOException { + try { + tsFile.writeLine(tsRecord); + } catch (WriteProcessException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/main/java/cn/edu/thu/tsfile/qp/common/SQLConstant.java b/src/main/java/cn/edu/thu/tsfile/qp/common/SQLConstant.java index 7b4b0b2..2b56082 100644 --- a/src/main/java/cn/edu/thu/tsfile/qp/common/SQLConstant.java +++ b/src/main/java/cn/edu/thu/tsfile/qp/common/SQLConstant.java @@ -1,7 +1,5 @@ package cn.edu.thu.tsfile.qp.common; -import cn.edu.thu.tsfile.timeseries.read.qp.Path; - import java.util.HashMap; import java.util.Map; @@ -12,21 +10,15 @@ * */ public class SQLConstant { - public static final String MREGE_EXTENSION = "merge"; - public static final String ERR_EXTENSION = "err"; public static final String PATH_SEPARATOR = "."; - public static final String PATH_SEPARATER_NO_REGEX = "\\."; - - public static final String DEFAULT_DELTA_OBJECT_TYPE = "defalut_delta_object_type"; - public static final String RESERVED_TIME = "time"; public static final String RESERVED_FREQ = "freq"; public static final String RESERVED_DELTA_OBJECT = "delta_object"; - public static final String IS_AGGREGATION = "IS_AGGREGATION"; - - public static final String lineFeedSignal = "\n"; - public static final String ROOT = "root"; - public static final String METADATA_PARAM_EQUAL = "="; + public static final String INT32 = "INT32"; + public static final String INT64 = "INT64"; + public static final String FLOAT = "FLOAT"; + public static final String DOUBLE = "DOUBLE"; + public static final String DEFAULT_ENCODING = "RLE"; public static final int KW_AND = 1; public static final int KW_OR = 2; @@ -47,9 +39,6 @@ public class SQLConstant { public static final int TOK_DELETE = 25; public static final int TOK_UPDATE = 26; public static final int TOK_QUERY = 27; - // public static final int TOK_VIRTUAL_TABLE = 30; - // public static final int TOK_TABNAME = 31; - // public static final int TOK_TABREF = 32; public static final int TOK_AUTHOR_CREATE = 41; public static final int TOK_AUTHOR_DROP = 42; @@ -134,7 +123,7 @@ public class SQLConstant { reverseWords.put(GREATERTHAN, LESSTHANOREQUALTO); } - public static boolean isReservedPath(Path pathStr) { + public static boolean isReservedPath(String pathStr) { return pathStr.equals(SQLConstant.RESERVED_TIME) || pathStr.equals(SQLConstant.RESERVED_FREQ) || pathStr.equals(SQLConstant.RESERVED_DELTA_OBJECT); diff --git a/src/main/scala/cn/edu/thu/tsfile/Converter.scala b/src/main/scala/cn/edu/thu/tsfile/Converter.scala index e59abb5..bc9a986 100755 --- a/src/main/scala/cn/edu/thu/tsfile/Converter.scala +++ b/src/main/scala/cn/edu/thu/tsfile/Converter.scala @@ -3,7 +3,7 @@ package cn.edu.thu.tsfile import java.util import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader -import cn.edu.thu.tsfile.file.metadata.enums.TSDataType +import cn.edu.thu.tsfile.file.metadata.enums.{TSDataType, TSEncoding} import cn.edu.thu.tsfile.io.HDFSInputStream import cn.edu.thu.tsfile.timeseries.read.metadata.SeriesSchema import cn.edu.thu.tsfile.timeseries.read.query.{QueryConfig, QueryEngine} @@ -14,6 +14,10 @@ import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ import cn.edu.thu.tsfile.qp.QueryProcessor import cn.edu.thu.tsfile.qp.common.{BasicOperator, FilterOperator, SQLConstant, TSQueryPlan} +import cn.edu.thu.tsfile.timeseries.write.desc.MeasurementDescriptor +import cn.edu.thu.tsfile.timeseries.write.record.{DataPoint, TSRecord} +import cn.edu.thu.tsfile.timeseries.write.schema.{FileSchema, SchemaBuilder} +import org.apache.spark.sql.Row import scala.collection.JavaConversions._ import scala.collection.mutable @@ -66,7 +70,7 @@ object Converter { * @param tsfileSchema all time series information in TSFile * @return sparkSQL table schema */ - def toSparkSqlSchema(tsfileSchema: util.ArrayList[SeriesSchema]): Option[StructType] = { + def toSqlSchema(tsfileSchema: util.ArrayList[SeriesSchema]): Option[StructType] = { val fields = new ListBuffer[StructField]() fields += StructField(SQLConstant.RESERVED_TIME, LongType, nullable = false) fields += StructField(SQLConstant.RESERVED_DELTA_OBJECT, StringType, nullable = false) @@ -94,6 +98,61 @@ object Converter { } } + + /** + * given a spark sql struct type, generate TsFile schema + * @param structType given sql schema + * @return TsFile schema + */ + def toTsFileSchema(structType: StructType, options: Map[String, String]): FileSchema = { + val schemaBuilder = new SchemaBuilder() + structType.fields.filter(f => { + !SQLConstant.isReservedPath(f.name) + }).foreach(f => { + val seriesSchema = getSeriesSchema(f, options) + schemaBuilder.addSeries(seriesSchema) + }) + schemaBuilder.build() + } + + + /** + * construct series schema from name and data type + * @param field series name + * @param options series data type + * @return series schema + */ + def getSeriesSchema(field: StructField, options: Map[String, String]): MeasurementDescriptor = { + val dataType = getTsDataType(field.dataType) + val encodingStr = dataType match { + case TSDataType.INT32 => options.getOrElse(SQLConstant.INT32, SQLConstant.DEFAULT_ENCODING) + case TSDataType.INT64 => options.getOrElse(SQLConstant.INT64, SQLConstant.DEFAULT_ENCODING) + case TSDataType.FLOAT => options.getOrElse(SQLConstant.FLOAT, SQLConstant.DEFAULT_ENCODING) + case TSDataType.DOUBLE => options.getOrElse(SQLConstant.DOUBLE, SQLConstant.DEFAULT_ENCODING) + case other => throw new UnsupportedOperationException(s"Unsupported type $other") + } + val encoding = TSEncoding.valueOf(encodingStr) + new MeasurementDescriptor(field.name, dataType, encoding, null) + } + + + /** + * return the TsFile data type of given spark sql data type + * @param dataType spark sql data type + * @return TsFile data type + */ + def getTsDataType(dataType: DataType): TSDataType = { + dataType match { + case IntegerType => TSDataType.INT32 + case LongType => TSDataType.INT64 + case BooleanType => TSDataType.BOOLEAN + case FloatType => TSDataType.FLOAT + case DoubleType => TSDataType.DOUBLE + case other => throw new UnsupportedOperationException(s"Unsupported type $other") + } + } + + /** * Use information given by sparkSQL to construct TSFile QueryConfigs for querying data. * @@ -116,7 +175,7 @@ object Converter { //remove invalid filters val validFilters = new ListBuffer[Filter]() filters.foreach {f => { - if(isValidFIlter(f)) + if(isValidFilter(f)) validFilters.add(f)} } @@ -152,16 +211,17 @@ object Converter { queryConfigs.toArray } - private def isValidFIlter(filter: Filter): Boolean = { + + private def isValidFilter(filter: Filter): Boolean = { filter match { case f: EqualTo => true case f: GreaterThan => true case f: GreaterThanOrEqual => true case f: LessThan => true case f: LessThanOrEqual => true - case f: Or => isValidFIlter(f.left) && isValidFIlter(f.right) - case f: And => isValidFIlter(f.left) && isValidFIlter(f.right) - case f: Not => isValidFIlter(f.child) + case f: Or => isValidFilter(f.left) && isValidFilter(f.right) + case f: And => isValidFilter(f.left) && isValidFilter(f.right) + case f: Not => isValidFilter(f.child) case _ => false } } @@ -169,8 +229,8 @@ object Converter { /** * Used in toQueryConfigs() to convert one query plan to one QueryConfig. * - * @param queryPlan TSFile logical query plan - * @return TSFile physical query plan + * @param queryPlan TsFile logical query plan + * @return TsFile physical query plan */ private def queryToConfig(queryPlan: TSQueryPlan): QueryConfig = { val selectedColumns = queryPlan.getPaths.toArray @@ -348,4 +408,39 @@ object Converter { case other => throw new UnsupportedOperationException(s"Unsupported type $other") } } + + + /** + * convert row to TSRecord + * @param row given spark sql row + * @return TSRecord + */ + def toTsRecord(row: Row): TSRecord = { + val schema = row.schema + val time = row.getAs[Long](SQLConstant.RESERVED_TIME) + val delta_object = row.getAs[String](SQLConstant.RESERVED_DELTA_OBJECT) + val tsRecord = new TSRecord(time, delta_object) + var i = 1 + schema.fields.filter(f => { + !SQLConstant.isReservedPath(f.name) + }).foreach(f => { + val name = f.name + val dataType = getTsDataType(f.dataType) + i = i + 1 + if (!row.isNullAt(i)) { + val value = f.dataType match { + case IntegerType => row.getAs[Int](name) + case LongType => row.getAs[Long](name) + case FloatType => row.getAs[Float](name) + case DoubleType => row.getAs[Double](name) + case other => throw new UnsupportedOperationException(s"Unsupported type $other") + } + val dataPoint = DataPoint.getDataPoint(dataType, name, value.toString) + tsRecord.addTuple(dataPoint) + } + }) + tsRecord + } + + } \ No newline at end of file diff --git a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala index 832a0c1..0dd5012 100755 --- a/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala +++ b/src/main/scala/cn/edu/thu/tsfile/DefaultSource.scala @@ -31,8 +31,8 @@ import scala.collection.mutable /** * TSFile data source * - * @author QJL - * @author MXW + * @author qiaojialin + * */ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { @@ -59,7 +59,7 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { //get union series in TsFile val tsfileSchema = Converter.getUnionSeries(files, conf) - Converter.toSparkSqlSchema(tsfileSchema) + Converter.toSqlSchema(tsfileSchema) } override def isSplitable( @@ -193,7 +193,7 @@ private[tsfile] class DefaultSource extends FileFormat with DataSourceRegister { job: Job, options: Map[String, String], dataSchema: StructType): OutputWriterFactory = { - new TsFileWriterFactory() + new TsFileWriterFactory(options) } } diff --git a/src/main/scala/cn/edu/thu/tsfile/TsFileOutputWriter.scala b/src/main/scala/cn/edu/thu/tsfile/TsFileOutputWriter.scala index dbe98dc..2f35b03 100644 --- a/src/main/scala/cn/edu/thu/tsfile/TsFileOutputWriter.scala +++ b/src/main/scala/cn/edu/thu/tsfile/TsFileOutputWriter.scala @@ -1,26 +1,32 @@ package cn.edu.thu.tsfile -import cn.edu.thu.tsfile.io.HDFSOutputStream -import cn.edu.thu.tsfile.timeseries.FileFormat.TsFile -import org.apache.hadoop.mapreduce.TaskAttemptContext +import cn.edu.thu.tsfile.io.TsFileOutputFormat +import cn.edu.thu.tsfile.timeseries.write.record.TSRecord +import org.apache.hadoop.io.NullWritable +import org.apache.hadoop.mapreduce.{RecordWriter, TaskAttemptContext} import org.apache.spark.sql.Row import org.apache.spark.sql.execution.datasources.OutputWriter -import org.json.JSONObject +import org.apache.spark.sql.types._ +private[tsfile] class TsFileOutputWriter( + pathStr: String, + dataSchema: StructType, + options: Map[String, String], + context: TaskAttemptContext) extends OutputWriter{ -private[tsfile] class TsFileOutputWriter(path: String, context: TaskAttemptContext) extends OutputWriter{ - - private val tsfile = { - val conf = context.getConfiguration - val hdfsOutput = new HDFSOutputStream(path, conf, true) - new TsFile(hdfsOutput, new JSONObject("")) + private val recordWriter: RecordWriter[NullWritable, TSRecord] = { + val fileSchema = Converter.toTsFileSchema(dataSchema, options) + new TsFileOutputFormat(fileSchema).getRecordWriter(context) } override def write(row: Row): Unit = { - tsfile.writeLine("") + if( row != null) { + val tsRecord = Converter.toTsRecord(row) + recordWriter.write(NullWritable.get(), tsRecord) + } } override def close(): Unit = { - tsfile.close() + recordWriter.close(context) } } diff --git a/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala b/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala index b64ef84..8f83ca5 100644 --- a/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala +++ b/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala @@ -7,17 +7,17 @@ import org.apache.spark.sql.types.StructType /** * @author qiaojialin */ -private[tsfile] class TsFileWriterFactory() extends OutputWriterFactory{ +private[tsfile] class TsFileWriterFactory(options: Map[String, String]) extends OutputWriterFactory{ override def newInstance( path: String, bucketId: Option[Int], dataSchema: StructType, context: TaskAttemptContext): OutputWriter = { - println("---newInstance---") - - println(dataSchema) + new TsFileOutputWriter(path, dataSchema, options, context) + } - new TsFileOutputWriter(path, context) + override def newWriter(path: String): OutputWriter = { + throw new UnsupportedOperationException("newInstance with just path not supported") } } diff --git a/src/test/resources/test.tsfile b/src/test/resources/test.tsfile new file mode 100644 index 0000000000000000000000000000000000000000..9522642be492d9a4e6259d48abe5da76e3b0e312 GIT binary patch literal 1233 zcmai!PfHvz7{)W1I2*@WANx)}rqBB{pe$HhZkCW;&+?t|k+!&U4puPiHwdI)0mfmZJwrCp%u3QoCf_PTV za!QudhiY>(V9oPdjg+{g&!ciQs`<&OpL_pq4^#mNTuztf35+5D{V}L6o@UAww^a`S za})6cP&(+j?!vv=fZ42=HTP7Em?q_9l?Dqo_Cp!8M9Ub&ct#BJ+#nYM|_f zFY~0RoRU2Zd^D^>^W8~_{78gUihsIQwYUQDHEx28M^;7}iLrx>xOLo_T2zJTm#al5 zmWV|nKtM)_RM^c3@lbbXr&nHI?UpyH@;8(DQ466#@BoVx11h=2z%-{}@X3S7lXgXZ zgK1I3Al-w8!C6jcJhZMu8&4SxgSh7mgRgiHl=;~bu}G9)VE0*)K}BBoF?gk*(X~bE GPwYPx=AaS) literal 0 HcmV?d00001 diff --git a/src/test/resources/tsfile/test1.tsfile b/src/test/resources/tsfile/test1.tsfile new file mode 100644 index 0000000000000000000000000000000000000000..9522642be492d9a4e6259d48abe5da76e3b0e312 GIT binary patch literal 1233 zcmai!PfHvz7{)W1I2*@WANx)}rqBB{pe$HhZkCW;&+?t|k+!&U4puPiHwdI)0mfmZJwrCp%u3QoCf_PTV za!QudhiY>(V9oPdjg+{g&!ciQs`<&OpL_pq4^#mNTuztf35+5D{V}L6o@UAww^a`S za})6cP&(+j?!vv=fZ42=HTP7Em?q_9l?Dqo_Cp!8M9Ub&ct#BJ+#nYM|_f zFY~0RoRU2Zd^D^>^W8~_{78gUihsIQwYUQDHEx28M^;7}iLrx>xOLo_T2zJTm#al5 zmWV|nKtM)_RM^c3@lbbXr&nHI?UpyH@;8(DQ466#@BoVx11h=2z%-{}@X3S7lXgXZ zgK1I3Al-w8!C6jcJhZMu8&4SxgSh7mgRgiHl=;~bu}G9)VE0*)K}BBoF?gk*(X~bE GPwYPx=AaS) literal 0 HcmV?d00001 diff --git a/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala b/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala index 68aff6d..8d65f17 100644 --- a/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala +++ b/src/test/scala/cn/edu/thu/tsfile/ConverterTest.scala @@ -81,7 +81,7 @@ class ConverterTest extends FunSuite with BeforeAndAfterAll { fields.add(new SeriesSchema("s4", TSDataType.DOUBLE, TSEncoding.PLAIN)) fields.add(new SeriesSchema("s5", TSDataType.BOOLEAN, TSEncoding.PLAIN)) fields.add(new SeriesSchema("s6", TSDataType.BYTE_ARRAY, TSEncoding.PLAIN)) - val sqlSchema = Converter.toSparkSqlSchema(fields) + val sqlSchema = Converter.toSqlSchema(fields) val expectedFields = Array( StructField(SQLConstant.RESERVED_TIME, LongType, nullable = false), diff --git a/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala index 7980e93..3adb7dd 100644 --- a/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala +++ b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala @@ -17,6 +17,7 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { private val tsfilePath1 = "src/test/resources/tsfile/test1.tsfile" private val tsfilePath2 = "src/test/resources/tsfile/test2.tsfile" private val outputPath = "src/test/resources/output" + private val outputPathFile = outputPath + "/part-m-00000" private var spark: SparkSession = _ override protected def beforeAll(): Unit = { @@ -34,15 +35,14 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { new CreateTSFile().createTSFile2(tsfilePath2) spark = SparkSession .builder() - .config("spark.master", "local") + .config("spark.master", "localhost") .appName("TSFile test") .getOrCreate() } override protected def afterAll(): Unit = { val out = new File(outputPath) - if (out.exists()) - out.delete() + deleteDir(out) try { spark.sparkContext.stop() } finally { @@ -50,20 +50,35 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { } } - test("writer") { + def deleteDir(dir: File): Unit = { + if(dir.isDirectory) { + dir.list().foreach(f => { + deleteDir(new File(dir, f)) + }) + } + dir.delete() + + } + + test("writer format") { val df = spark.read.tsfile(tsfilePath1) -// val df = spark.createDataFrame( -// Seq( -// ("root.car.d1", 8, 1, 9.8), -// ("root.car.d1", 8, 2, 8.7), -// ("root.car.d2", 7, 3, 5.5), -// ("root.car.d2", 7, 4, 2.0)) -// ).toDF("delta_object", "timestamp", "sensor1", "sensor2") + df.show() + df.write.format("cn.edu.thu.tsfile").save(outputPath) + val newDf = spark.read.tsfile(outputPathFile) + newDf.show() + Assert.assertEquals(newDf.collectAsList(), df.collectAsList()) + } + test("writer") { + val df = spark.read.tsfile(tsfilePath1) df.show() df.write.tsfile(outputPath) + val newDf = spark.read.tsfile(outputPathFile) + newDf.show() + Assert.assertEquals(newDf.collectAsList(), df.collectAsList()) } + test("tsfile_qp") { val df = spark.read.tsfile(tsfileFolder) df.createOrReplaceTempView("tsfile_table") From d75865c6e81df2045ced006ab0ae15027b25104c Mon Sep 17 00:00:00 2001 From: qiaojialingithub <646274302@qq.com> Date: Mon, 26 Jun 2017 17:32:59 +0800 Subject: [PATCH 15/15] change run mode --- src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala | 4 ---- src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala b/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala index 8f83ca5..1c79083 100644 --- a/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala +++ b/src/main/scala/cn/edu/thu/tsfile/TsFileWriterFactory.scala @@ -16,8 +16,4 @@ private[tsfile] class TsFileWriterFactory(options: Map[String, String]) extends context: TaskAttemptContext): OutputWriter = { new TsFileOutputWriter(path, dataSchema, options, context) } - - override def newWriter(path: String): OutputWriter = { - throw new UnsupportedOperationException("newInstance with just path not supported") - } } diff --git a/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala index 3adb7dd..5061d54 100644 --- a/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala +++ b/src/test/scala/cn/edu/thu/tsfile/TSFileSuit.scala @@ -35,7 +35,7 @@ class TSFileSuit extends FunSuite with BeforeAndAfterAll { new CreateTSFile().createTSFile2(tsfilePath2) spark = SparkSession .builder() - .config("spark.master", "localhost") + .config("spark.master", "local") .appName("TSFile test") .getOrCreate() }