diff --git a/.gitignore b/.gitignore index fa2e3ec24..abe15d9c8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ -/.idea/** /.mvn/.gradle-enterprise/gradle-enterprise-workspace-id /.mvn/wrapper/maven-wrapper.jar **/target/** /tsfile/test.tsfile + +# intellij IDE files +**/*.iml +**/.idea/ +**/*.log +**/*.ipr +**/*.iws diff --git a/common-api/pom.xml b/common/pom.xml similarity index 98% rename from common-api/pom.xml rename to common/pom.xml index 45d6e1b04..6a384f67e 100644 --- a/common-api/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ tsfile-parent 1.0.0-SNAPSHOT - common-api + common TsFile: Common API diff --git a/common-api/src/main/java/org/apache/tsfile/block/TsBlockBuilderStatus.java b/common/src/main/java/org/apache/tsfile/block/TsBlockBuilderStatus.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/block/TsBlockBuilderStatus.java rename to common/src/main/java/org/apache/tsfile/block/TsBlockBuilderStatus.java diff --git a/common-api/src/main/java/org/apache/tsfile/block/column/Column.java b/common/src/main/java/org/apache/tsfile/block/column/Column.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/block/column/Column.java rename to common/src/main/java/org/apache/tsfile/block/column/Column.java diff --git a/common-api/src/main/java/org/apache/tsfile/block/column/ColumnBuilder.java b/common/src/main/java/org/apache/tsfile/block/column/ColumnBuilder.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/block/column/ColumnBuilder.java rename to common/src/main/java/org/apache/tsfile/block/column/ColumnBuilder.java diff --git a/common-api/src/main/java/org/apache/tsfile/block/column/ColumnBuilderStatus.java b/common/src/main/java/org/apache/tsfile/block/column/ColumnBuilderStatus.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/block/column/ColumnBuilderStatus.java rename to common/src/main/java/org/apache/tsfile/block/column/ColumnBuilderStatus.java diff --git a/common-api/src/main/java/org/apache/tsfile/block/column/ColumnEncoding.java b/common/src/main/java/org/apache/tsfile/block/column/ColumnEncoding.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/block/column/ColumnEncoding.java rename to common/src/main/java/org/apache/tsfile/block/column/ColumnEncoding.java diff --git a/common-api/src/main/java/org/apache/tsfile/enums/TSDataType.java b/common/src/main/java/org/apache/tsfile/enums/TSDataType.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/enums/TSDataType.java rename to common/src/main/java/org/apache/tsfile/enums/TSDataType.java diff --git a/common-api/src/main/java/org/apache/tsfile/utils/Binary.java b/common/src/main/java/org/apache/tsfile/utils/Binary.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/utils/Binary.java rename to common/src/main/java/org/apache/tsfile/utils/Binary.java diff --git a/common-api/src/main/java/org/apache/tsfile/utils/Constants.java b/common/src/main/java/org/apache/tsfile/utils/Constants.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/utils/Constants.java rename to common/src/main/java/org/apache/tsfile/utils/Constants.java diff --git a/common-api/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java b/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java rename to common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java diff --git a/common-api/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java b/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java rename to common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java diff --git a/common-api/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java b/common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java similarity index 100% rename from common-api/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java rename to common/src/main/java/org/apache/tsfile/write/UnSupportedDataTypeException.java diff --git a/examples/pom.xml b/examples/pom.xml new file mode 100644 index 000000000..ca7e92158 --- /dev/null +++ b/examples/pom.xml @@ -0,0 +1,77 @@ + + + + 4.0.0 + + org.apache.tsfile + tsfile-parent + 1.0.0-SNAPSHOT + + tsfile-examples + pom + TSFile: Examples + + + org.apache.tsfile + tsfile + 1.0.0-SNAPSHOT + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + org.apache.maven.plugins + maven-dependency-plugin + + + check-dependencies + + analyze-only + + verify + + true + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + true + + + + + + diff --git a/examples/readme.md b/examples/readme.md new file mode 100644 index 000000000..d6907478d --- /dev/null +++ b/examples/readme.md @@ -0,0 +1,84 @@ + + +# Function +``` +The example is to show how to write and read a TsFile File. +``` +# Usage +## Dependencies with Maven + +``` + + + org.apache.tsfile + tsfile + 1.0.0-SNAPSHOT + + +``` + + +## Run TsFileWrite.java + + + This class is to show how to write a TsFile. It provided two ways: + + The first one is using a JSON string for measurement(s). The JSON string is an array of JSON + objects(schema). The object must include the *measurement_id*, *datatype*, *encoding*, and + *compressor*. + + An example JSON string is provided in the comments in + the method + + public static void tsFileWriteWithJson() throws IOException,WriteProcessException + It uses this interface + + public void addMeasurementByJson(JSONObject measurement) throws WriteProcessException + An alternative way is to add these measurements directly(manually) by the second interface: + + public void addMeasurement(MeasurementSchema measurementSchema) throws WriteProcessException + + The method + + public static void tsFileWriteDirect() throws IOException,WriteProcessException + shows how to use that interface. + + Note that the measurements in the two methods are the same therefore there output TsFile should also be identical. + + + +## Run TsFileRead.java + + This class is to show how to read TsFile file named "testDirect.tsfile". + + The TsFile file "testDirect.tsfile" is generated from class TsFileWrite. + + It generates the same TsFile(testDirect.tsfile and testWithJson.tsfile) file by two different ways + + Run TsFileWrite to generate the testDirect.tsfile first + +## Run TsFileSequenceRead.java + + This class is to show the structure of a TsFile. + +### Notice +  For detail, please refer to https://github.com/apache/iotdb/blob/master/tsfile/README.md. diff --git a/examples/src/main/java/org/apache/tsfile/Constant.java b/examples/src/main/java/org/apache/tsfile/Constant.java new file mode 100644 index 000000000..a6a5bc955 --- /dev/null +++ b/examples/src/main/java/org/apache/tsfile/Constant.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile; + +public class Constant { + + private Constant() {} + + static final String SENSOR_1 = "sensor_1"; + static final String SENSOR_2 = "sensor_2"; + static final String SENSOR_3 = "sensor_3"; + + static final String DEVICE_PREFIX = "device_"; + static final String DEVICE_1 = "root.sg.device_1"; + static final String DEVICE_2 = "root.sg.device_2"; +} diff --git a/examples/src/main/java/org/apache/tsfile/TsFileForceAppendWrite.java b/examples/src/main/java/org/apache/tsfile/TsFileForceAppendWrite.java new file mode 100644 index 000000000..4bea2092e --- /dev/null +++ b/examples/src/main/java/org/apache/tsfile/TsFileForceAppendWrite.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.LongDataPoint; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.ForceAppendTsFileWriter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +public class TsFileForceAppendWrite { + + private static final Logger LOGGER = LoggerFactory.getLogger(TsFileForceAppendWrite.class); + + public static void main(String[] args) throws IOException { + String path = "test.tsfile"; + File f = FSFactoryProducer.getFSFactory().getFile(path); + if (f.exists()) { + Files.delete(f.toPath()); + } + + try (TsFileWriter tsFileWriter = new TsFileWriter(f)) { + + // add measurements into file schema + for (int i = 0; i < 4; i++) { + tsFileWriter.registerTimeseries( + new Path(Constant.DEVICE_PREFIX + i), + new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.RLE)); + tsFileWriter.registerTimeseries( + new Path(Constant.DEVICE_PREFIX + i), + new MeasurementSchema(Constant.SENSOR_2, TSDataType.INT64, TSEncoding.RLE)); + tsFileWriter.registerTimeseries( + new Path(Constant.DEVICE_PREFIX + i), + new MeasurementSchema(Constant.SENSOR_3, TSDataType.INT64, TSEncoding.RLE)); + } + + // construct TSRecord + for (int i = 0; i < 100; i++) { + TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX + (i % 4)); + DataPoint dPoint1 = new LongDataPoint(Constant.SENSOR_1, i); + DataPoint dPoint2 = new LongDataPoint(Constant.SENSOR_2, i); + DataPoint dPoint3 = new LongDataPoint(Constant.SENSOR_3, i); + tsRecord.addTuple(dPoint1); + tsRecord.addTuple(dPoint2); + tsRecord.addTuple(dPoint3); + + // write TSRecord + tsFileWriter.write(tsRecord); + } + } catch (Exception e) { + LOGGER.error("meet error in TsFileWrite ", e); + } + + // open the closed file with ForceAppendTsFileWriter + + try (ForceAppendTsFileWriter fwriter = new ForceAppendTsFileWriter(f)) { + fwriter.doTruncate(); + write(fwriter); + } catch (Exception e) { + LOGGER.error("ForceAppendTsFileWriter truncate or write error ", e); + } + } + + private static void write(ForceAppendTsFileWriter fwriter) { + try (TsFileWriter tsFileWriter1 = new TsFileWriter(fwriter)) { + // add measurements into file schema + for (int i = 0; i < 4; i++) { + tsFileWriter1.registerTimeseries( + new Path(Constant.DEVICE_PREFIX + i), + new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.RLE)); + tsFileWriter1.registerTimeseries( + new Path(Constant.DEVICE_PREFIX + i), + new MeasurementSchema(Constant.SENSOR_2, TSDataType.INT64, TSEncoding.RLE)); + tsFileWriter1.registerTimeseries( + new Path(Constant.DEVICE_PREFIX + i), + new MeasurementSchema(Constant.SENSOR_3, TSDataType.INT64, TSEncoding.RLE)); + } + // construct TSRecord + for (int i = 100; i < 120; i++) { + TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX + (i % 4)); + DataPoint dPoint1 = new LongDataPoint(Constant.SENSOR_1, i); + DataPoint dPoint2 = new LongDataPoint(Constant.SENSOR_2, i); + DataPoint dPoint3 = new LongDataPoint(Constant.SENSOR_3, i); + tsRecord.addTuple(dPoint1); + tsRecord.addTuple(dPoint2); + tsRecord.addTuple(dPoint3); + + // write TSRecord + tsFileWriter1.write(tsRecord); + } + } catch (Exception e) { + LOGGER.error("meet error in TsFileWrite ", e); + } + } +} diff --git a/examples/src/main/java/org/apache/tsfile/TsFileRead.java b/examples/src/main/java/org/apache/tsfile/TsFileRead.java new file mode 100644 index 000000000..cb94e9b68 --- /dev/null +++ b/examples/src/main/java/org/apache/tsfile/TsFileRead.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile; + +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.IExpression; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.expression.impl.BinaryExpression; +import org.apache.tsfile.read.expression.impl.GlobalTimeExpression; +import org.apache.tsfile.read.expression.impl.SingleSeriesExpression; +import org.apache.tsfile.read.filter.factory.TimeFilterApi; +import org.apache.tsfile.read.filter.factory.ValueFilterApi; +import org.apache.tsfile.read.query.dataset.QueryDataSet; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.apache.tsfile.Constant.DEVICE_1; +import static org.apache.tsfile.Constant.SENSOR_1; +import static org.apache.tsfile.Constant.SENSOR_2; +import static org.apache.tsfile.Constant.SENSOR_3; + +/** + * The class is to show how to read TsFile file named "test.tsfile". The TsFile file "test.tsfile" + * is generated from class TsFileWriteWithTSRecord or TsFileWriteWithTablet. Run + * TsFileWriteWithTSRecord or TsFileWriteWithTablet to generate the test.tsfile first + */ +public class TsFileRead { + + private static final Logger LOGGER = LoggerFactory.getLogger(TsFileRead.class); + + private static void queryAndPrint( + ArrayList paths, TsFileReader readTsFile, IExpression statement) throws IOException { + QueryExpression queryExpression = QueryExpression.create(paths, statement); + QueryDataSet queryDataSet = readTsFile.query(queryExpression); + while (queryDataSet.hasNext()) { + String next = queryDataSet.next().toString(); + LOGGER.info(next); + } + LOGGER.info("----------------"); + } + + public static void main(String[] args) throws IOException { + + // file path + String path = "test.tsfile"; + + // create reader and get the readTsFile interface + try (TsFileSequenceReader reader = new TsFileSequenceReader(path); + TsFileReader readTsFile = new TsFileReader(reader)) { + + // use these paths(all measurements) for all the queries + ArrayList paths = new ArrayList<>(); + paths.add(new Path(DEVICE_1, SENSOR_1, true)); + paths.add(new Path(DEVICE_1, SENSOR_2, true)); + paths.add(new Path(DEVICE_1, SENSOR_3, true)); + + // no filter, should select 1 2 3 4 6 7 8 + queryAndPrint(paths, readTsFile, null); + + // time filter : 4 <= time <= 10, should select 4 6 7 8 + IExpression timeFilter = + BinaryExpression.and( + new GlobalTimeExpression(TimeFilterApi.gtEq(4L)), + new GlobalTimeExpression(TimeFilterApi.ltEq(10L))); + queryAndPrint(paths, readTsFile, timeFilter); + + // value filter : device_1.sensor_2 <= 20, should select 1 2 4 6 7 + IExpression valueFilter = + new SingleSeriesExpression(new Path(DEVICE_1, SENSOR_2, true), ValueFilterApi.ltEq(20L)); + queryAndPrint(paths, readTsFile, valueFilter); + + // time filter : 4 <= time <= 10, value filter : device_1.sensor_3 >= 20, should select 4 7 8 + timeFilter = + BinaryExpression.and( + new GlobalTimeExpression(TimeFilterApi.gtEq(4L)), + new GlobalTimeExpression(TimeFilterApi.ltEq(10L))); + valueFilter = + new SingleSeriesExpression(new Path(DEVICE_1, SENSOR_3, true), ValueFilterApi.gtEq(20L)); + IExpression finalFilter = BinaryExpression.and(timeFilter, valueFilter); + queryAndPrint(paths, readTsFile, finalFilter); + } + } +} diff --git a/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java b/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java new file mode 100644 index 000000000..132d1bb4d --- /dev/null +++ b/examples/src/main/java/org/apache/tsfile/TsFileSequenceRead.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.MetaMarker; +import org.apache.tsfile.file.header.ChunkGroupHeader; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.reader.page.PageReader; +import org.apache.tsfile.read.reader.page.TimePageReader; +import org.apache.tsfile.read.reader.page.ValuePageReader; +import org.apache.tsfile.utils.TsPrimitiveType; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** This tool is used to read TsFile sequentially, including nonAligned or aligned timeseries. */ +public class TsFileSequenceRead { + // if you wanna print detailed datas in pages, then turn it true. + private static boolean printDetail = false; + public static final String POINT_IN_PAGE = "\t\tpoints in the page: "; + + @SuppressWarnings({ + "squid:S3776", + "squid:S106" + }) // Suppress high Cognitive Complexity and Standard outputs warning + public static void main(String[] args) throws IOException { + String filename = "test.tsfile"; + if (args.length >= 1) { + filename = args[0]; + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(filename)) { + System.out.println( + "file length: " + FSFactoryProducer.getFSFactory().getFile(filename).length()); + System.out.println("file magic head: " + reader.readHeadMagic()); + System.out.println("file magic tail: " + reader.readTailMagic()); + System.out.println("Level 1 metadata position: " + reader.getFileMetadataPos()); + System.out.println("Level 1 metadata size: " + reader.getTsFileMetadataSize()); + // Sequential reading of one ChunkGroup now follows this order: + // first the CHUNK_GROUP_HEADER, then SeriesChunks (headers and data) in one ChunkGroup + // Because we do not know how many chunks a ChunkGroup may have, we should read one byte (the + // marker) ahead and judge accordingly. + reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); + System.out.println("position: " + reader.position()); + List timeBatch = new ArrayList<>(); + int pageIndex = 0; + byte marker; + while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) { + switch (marker) { + case MetaMarker.CHUNK_HEADER: + case MetaMarker.TIME_CHUNK_HEADER: + case MetaMarker.VALUE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: + System.out.println("\t[Chunk]"); + System.out.println("\tchunk type: " + marker); + System.out.println("\tposition: " + reader.position()); + ChunkHeader header = reader.readChunkHeader(marker); + System.out.println("\tMeasurement: " + header.getMeasurementID()); + if (header.getDataSize() == 0) { + // empty value chunk + System.out.println("\t-- Empty Chunk "); + break; + } + System.out.println( + "\tChunk Size: " + (header.getDataSize() + header.getSerializedSize())); + Decoder defaultTimeDecoder = + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64); + Decoder valueDecoder = + Decoder.getDecoderByType(header.getEncodingType(), header.getDataType()); + int dataSize = header.getDataSize(); + pageIndex = 0; + if (header.getDataType() == TSDataType.VECTOR) { + timeBatch.clear(); + } + while (dataSize > 0) { + valueDecoder.reset(); + System.out.println( + "\t\t[Page" + pageIndex + "]\n \t\tPage head position: " + reader.position()); + PageHeader pageHeader = + reader.readPageHeader( + header.getDataType(), + (header.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + System.out.println("\t\tPage data position: " + reader.position()); + ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType()); + System.out.println( + "\t\tUncompressed page data size: " + pageHeader.getUncompressedSize()); + System.out.println( + "\t\tCompressed page data size: " + pageHeader.getCompressedSize()); + if ((header.getChunkType() & TsFileConstant.TIME_COLUMN_MASK) + == TsFileConstant.TIME_COLUMN_MASK) { // Time Chunk + TimePageReader timePageReader = + new TimePageReader(pageHeader, pageData, defaultTimeDecoder); + timeBatch.add(timePageReader.getNextTimeBatch()); + System.out.println(POINT_IN_PAGE + timeBatch.get(pageIndex).length); + if (printDetail) { + for (int i = 0; i < timeBatch.get(pageIndex).length; i++) { + System.out.println("\t\t\ttime: " + timeBatch.get(pageIndex)[i]); + } + } + } else if ((header.getChunkType() & TsFileConstant.VALUE_COLUMN_MASK) + == TsFileConstant.VALUE_COLUMN_MASK) { // Value Chunk + ValuePageReader valuePageReader = + new ValuePageReader(pageHeader, pageData, header.getDataType(), valueDecoder); + TsPrimitiveType[] valueBatch = + valuePageReader.nextValueBatch(timeBatch.get(pageIndex)); + if (valueBatch.length == 0) { + System.out.println("\t\t-- Empty Page "); + } else { + System.out.println(POINT_IN_PAGE + valueBatch.length); + } + if (printDetail) { + for (TsPrimitiveType batch : valueBatch) { + System.out.println("\t\t\tvalue: " + batch); + } + } + } else { // NonAligned Chunk + PageReader pageReader = + new PageReader( + pageData, header.getDataType(), valueDecoder, defaultTimeDecoder); + BatchData batchData = pageReader.getAllSatisfiedPageData(); + if (header.getChunkType() == MetaMarker.CHUNK_HEADER) { + System.out.println(POINT_IN_PAGE + pageHeader.getNumOfValues()); + } else { + System.out.println(POINT_IN_PAGE + batchData.length()); + } + if (printDetail) { + while (batchData.hasCurrent()) { + System.out.println( + "\t\t\ttime, value: " + + batchData.currentTime() + + ", " + + batchData.currentValue()); + batchData.next(); + } + } + } + pageIndex++; + dataSize -= pageHeader.getSerializedPageSize(); + } + break; + case MetaMarker.CHUNK_GROUP_HEADER: + System.out.println("[Chunk Group]"); + System.out.println("Chunk Group Header position: " + reader.position()); + ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader(); + System.out.println("device: " + chunkGroupHeader.getDeviceID()); + break; + case MetaMarker.OPERATION_INDEX_RANGE: + reader.readPlanIndex(); + System.out.println("minPlanIndex: " + reader.getMinPlanIndex()); + System.out.println("maxPlanIndex: " + reader.getMaxPlanIndex()); + break; + default: + MetaMarker.handleUnexpectedMarker(marker); + } + } + System.out.println("[Metadata]"); + for (String device : reader.getAllDevices()) { + Map> seriesMetaData = reader.readChunkMetadataInDevice(device); + System.out.printf( + "\t[Device]Device %s, Number of Measurements %d%n", device, seriesMetaData.size()); + for (Map.Entry> serie : seriesMetaData.entrySet()) { + System.out.println("\t\tMeasurement:" + serie.getKey()); + for (ChunkMetadata chunkMetadata : serie.getValue()) { + System.out.println("\t\tFile offset:" + chunkMetadata.getOffsetOfChunkHeader()); + } + } + } + } + } +} diff --git a/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java b/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java new file mode 100644 index 000000000..d218fc945 --- /dev/null +++ b/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.LongDataPoint; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +public class TsFileWriteAlignedWithTSRecord { + private static final Logger logger = + LoggerFactory.getLogger(TsFileWriteAlignedWithTSRecord.class); + + public static void main(String[] args) throws IOException { + File f = FSFactoryProducer.getFSFactory().getFile("alignedRecord.tsfile"); + if (f.exists()) { + try { + Files.delete(f.toPath()); + } catch (IOException e) { + throw new IOException("can not delete " + f.getAbsolutePath()); + } + } + + try (TsFileWriter tsFileWriter = new TsFileWriter(f)) { + List measurementSchemas = new ArrayList<>(); + measurementSchemas.add( + new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.RLE)); + measurementSchemas.add( + new MeasurementSchema(Constant.SENSOR_2, TSDataType.INT64, TSEncoding.RLE)); + measurementSchemas.add( + new MeasurementSchema(Constant.SENSOR_3, TSDataType.INT64, TSEncoding.RLE)); + + // register timeseries + tsFileWriter.registerAlignedTimeseries(new Path(Constant.DEVICE_1), measurementSchemas); + + // example1 + writeAligned(tsFileWriter, Constant.DEVICE_1, measurementSchemas, 1000000, 0, 0); + } catch (WriteProcessException e) { + logger.error("write TSRecord failed", e); + } + } + + private static void writeAligned( + TsFileWriter tsFileWriter, + String deviceId, + List schemas, + long rowSize, + long startTime, + long startValue) + throws IOException, WriteProcessException { + for (long time = startTime; time < rowSize + startTime; time++) { + // construct TsRecord + TSRecord tsRecord = new TSRecord(time, deviceId); + for (IMeasurementSchema schema : schemas) { + DataPoint dPoint = new LongDataPoint(schema.getMeasurementId(), startValue++); + tsRecord.addTuple(dPoint); + } + // write + tsFileWriter.writeAligned(tsRecord); + } + } +} diff --git a/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java b/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java new file mode 100644 index 000000000..d82acc62e --- /dev/null +++ b/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.tsfile.Constant.DEVICE_1; +import static org.apache.tsfile.Constant.DEVICE_2; +import static org.apache.tsfile.Constant.SENSOR_1; +import static org.apache.tsfile.Constant.SENSOR_2; +import static org.apache.tsfile.Constant.SENSOR_3; + +public class TsFileWriteAlignedWithTablet { + private static final Logger LOGGER = LoggerFactory.getLogger(TsFileWriteAlignedWithTablet.class); + + public static void main(String[] args) throws IOException { + File f = FSFactoryProducer.getFSFactory().getFile("alignedTablet.tsfile"); + if (f.exists()) { + try { + Files.delete(f.toPath()); + } catch (IOException e) { + throw new IOException("can not delete " + f.getAbsolutePath()); + } + } + + try (TsFileWriter tsFileWriter = new TsFileWriter(f)) { + List measurementSchemas = new ArrayList<>(); + measurementSchemas.add(new MeasurementSchema(SENSOR_1, TSDataType.TEXT, TSEncoding.PLAIN)); + measurementSchemas.add(new MeasurementSchema(SENSOR_2, TSDataType.TEXT, TSEncoding.PLAIN)); + measurementSchemas.add(new MeasurementSchema(SENSOR_3, TSDataType.TEXT, TSEncoding.PLAIN)); + + // register align timeseries + tsFileWriter.registerAlignedTimeseries(new Path(DEVICE_1), measurementSchemas); + + // example 1 + writeAlignedWithTablet(tsFileWriter, DEVICE_1, measurementSchemas, 200000, 0, 0); + + writeNonAlignedWithTablet(tsFileWriter); // write nonAligned timeseries + } catch (WriteProcessException e) { + LOGGER.error("write Tablet failed", e); + } + } + + private static void writeAlignedWithTablet( + TsFileWriter tsFileWriter, + String deviceId, + List schemas, + long rowNum, + long startTime, + long startValue) + throws IOException, WriteProcessException { + Tablet tablet = new Tablet(deviceId, schemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + long sensorNum = schemas.size(); + + for (long r = 0; r < rowNum; r++, startValue++) { + int row = tablet.rowSize++; + timestamps[row] = startTime++; + for (int i = 0; i < sensorNum; i++) { + Binary[] textSensor = (Binary[]) values[i]; + textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + } + // write + if (tablet.rowSize == tablet.getMaxRowNumber()) { + tsFileWriter.writeAligned(tablet); + tablet.reset(); + } + } + // write + if (tablet.rowSize != 0) { + tsFileWriter.writeAligned(tablet); + tablet.reset(); + } + } + + private static void writeNonAlignedWithTablet(TsFileWriter tsFileWriter) + throws WriteProcessException, IOException { + // register nonAlign timeseries + tsFileWriter.registerTimeseries( + new Path(DEVICE_2), new MeasurementSchema(SENSOR_1, TSDataType.INT64, TSEncoding.RLE)); + tsFileWriter.registerTimeseries( + new Path(DEVICE_2), new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE)); + // construct Tablet + List measurementSchemas = new ArrayList<>(); + measurementSchemas.add(new MeasurementSchema(SENSOR_1, TSDataType.INT64, TSEncoding.RLE)); + measurementSchemas.add(new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE)); + Tablet tablet = new Tablet(DEVICE_2, measurementSchemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + int rowNum = 100; + int sensorNum = measurementSchemas.size(); + long timestamp = 1; + long value = 1000000L; + for (int r = 0; r < rowNum; r++, value++) { + int row = tablet.rowSize++; + timestamps[row] = timestamp++; + for (int i = 0; i < sensorNum; i++) { + long[] sensor = (long[]) values[i]; + sensor[row] = value; + } + // write + if (tablet.rowSize == tablet.getMaxRowNumber()) { + tsFileWriter.write(tablet); + tablet.reset(); + } + } + // write + if (tablet.rowSize != 0) { + tsFileWriter.write(tablet); + tablet.reset(); + } + } +} diff --git a/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java b/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java new file mode 100644 index 000000000..3d0d7ff75 --- /dev/null +++ b/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.LongDataPoint; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +/** + * An example of writing data with TSRecord to TsFile It uses the interface: public void + * addMeasurement(MeasurementSchema measurementSchema) throws WriteProcessException + */ +public class TsFileWriteWithTSRecord { + private static final Logger LOGGER = LoggerFactory.getLogger(TsFileWriteWithTSRecord.class); + + public static void main(String[] args) { + try { + String path = "Record.tsfile"; + File f = FSFactoryProducer.getFSFactory().getFile(path); + if (f.exists()) { + Files.delete(f.toPath()); + } + + try (TsFileWriter tsFileWriter = new TsFileWriter(f)) { + List schemas = new ArrayList<>(); + schemas.add(new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.RLE)); + schemas.add(new MeasurementSchema(Constant.SENSOR_2, TSDataType.INT64, TSEncoding.RLE)); + schemas.add(new MeasurementSchema(Constant.SENSOR_3, TSDataType.INT64, TSEncoding.RLE)); + + // register timeseries + tsFileWriter.registerTimeseries(new Path(Constant.DEVICE_1), schemas); + + // example1 + write(tsFileWriter, Constant.DEVICE_1, schemas, 10000, 0, 0); + } + } catch (Exception e) { + LOGGER.error("TsFileWriteWithTSRecord meet error", e); + } + } + + private static void write( + TsFileWriter tsFileWriter, + String deviceId, + List schemas, + long rowSize, + long startTime, + long startValue) + throws IOException, WriteProcessException { + for (long time = startTime; time < rowSize + startTime; time++) { + // construct TsRecord + TSRecord tsRecord = new TSRecord(time, deviceId); + for (IMeasurementSchema schema : schemas) { + DataPoint dPoint = new LongDataPoint(schema.getMeasurementId(), startValue++); + tsRecord.addTuple(dPoint); + } + // write + tsFileWriter.write(tsRecord); + } + } +} diff --git a/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java b/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java new file mode 100644 index 000000000..2461f1da0 --- /dev/null +++ b/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +/** An example of writing data with Tablet to TsFile */ +public class TsFileWriteWithTablet { + + private static final Logger LOGGER = LoggerFactory.getLogger(TsFileWriteWithTablet.class); + + public static void main(String[] args) { + try { + String path = "Tablet.tsfile"; + File f = FSFactoryProducer.getFSFactory().getFile(path); + if (f.exists()) { + Files.delete(f.toPath()); + } + + try (TsFileWriter tsFileWriter = new TsFileWriter(f)) { + List measurementSchemas = new ArrayList<>(); + measurementSchemas.add( + new MeasurementSchema(Constant.SENSOR_1, TSDataType.TEXT, TSEncoding.PLAIN)); + measurementSchemas.add( + new MeasurementSchema(Constant.SENSOR_2, TSDataType.TEXT, TSEncoding.PLAIN)); + measurementSchemas.add( + new MeasurementSchema(Constant.SENSOR_3, TSDataType.TEXT, TSEncoding.PLAIN)); + + // register nonAligned timeseries + tsFileWriter.registerTimeseries(new Path(Constant.DEVICE_1), measurementSchemas); + + // example 1 + writeWithTablet(tsFileWriter, Constant.DEVICE_1, measurementSchemas, 10000, 0, 0); + } + } catch (Exception e) { + LOGGER.error("meet error in TsFileWrite with tablet", e); + } + } + + private static void writeWithTablet( + TsFileWriter tsFileWriter, + String deviceId, + List schemas, + long rowNum, + long startTime, + long startValue) + throws IOException, WriteProcessException { + Tablet tablet = new Tablet(deviceId, schemas); + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + long sensorNum = schemas.size(); + + for (long r = 0; r < rowNum; r++, startValue++) { + int row = tablet.rowSize++; + timestamps[row] = startTime++; + for (int i = 0; i < sensorNum; i++) { + Binary[] textSensor = (Binary[]) values[i]; + textSensor[row] = new Binary("testString.........", TSFileConfig.STRING_CHARSET); + } + // write + if (tablet.rowSize == tablet.getMaxRowNumber()) { + tsFileWriter.write(tablet); + tablet.reset(); + } + } + // write + if (tablet.rowSize != 0) { + tsFileWriter.write(tablet); + tablet.reset(); + } + } +} diff --git a/pom.xml b/pom.xml index 96637b8a0..6ee08e6c3 100644 --- a/pom.xml +++ b/pom.xml @@ -32,8 +32,9 @@ pom Apache TsFile Project Parent POM - common-api + common tsfile + examples 1.8 diff --git a/tsfile/pom.xml b/tsfile/pom.xml index 4c76fa893..8596c250f 100644 --- a/tsfile/pom.xml +++ b/tsfile/pom.xml @@ -37,7 +37,7 @@ org.apache.tsfile - common-api + common 1.0.0-SNAPSHOT