rewrite some Tsfile C++ interface (#304)

* rewrite some Tsfile C++ interface * fix format * fix the problem in test/CMakeLists * add algorith and memory in tablet.h * add some header file * fix memory leak * fix for CWrapper and timeFilte * fix for ut * Add parser for path and fix for get device name * fix complie error for ubuntu and windows * add path test add fix complie error * Remove redundant tests * fix for windows * fix bug for complie when using mingw and change name parser/generated * fix include name * add download uuid in linux autoly * fix warning * fix bug in ubuntu * try to fix link bug in windows * try to fix bug in windows * output debug detail * Removed unnecessary include * link static lib instead of shared lib * remove static crt * debug detail * fix for bug in windows * Add compilation details on windows in Readme
apache · Dec 18, 2024 · 39c32d7 · 39c32d7
1 parent 0579fc7
commit 39c32d7
Show file tree

Hide file tree

Showing 438 changed files with 48,265 additions and 443 deletions.
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -21,7 +21,6 @@ project(TsFile_CPP)
 
 cmake_policy(SET CMP0079 NEW)
 set(TsFile_CPP_VERSION 1.2.0.dev)
-
 set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -Werror")
 message("cmake using: USE_CPP11=${USE_CPP11}")
 
@@ -76,6 +75,8 @@ set(PROJECT_INCLUDE_DIR ${PROJECT_INCLUDE_DIR}
 set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
 include_directories(${PROJECT_INCLUDE_DIR})
 
+include_directories(${PROJECT_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src)
+
 add_subdirectory(third_party)
 
 add_subdirectory(src)

diff --git a/cpp/README.md b/cpp/README.md
@@ -74,6 +74,13 @@ To build tsfile, you can run: `bash build.sh`. If you have Maven tools, you can
 
 Before you submit your code to GitHub, please ensure that the `mvn` compilation is correct.
 
+If you compile using MinGW on windows and encounter an error, you can try replacing MinGW with the following version that we have tried without problems:
+
+* GCC 14.2.0 (with **POSIX** threads) + LLVM/Clang/LLD/LLDB 18.1.8 + MinGW-w64 12.0.0 UCRT - release 1
+* GCC 12.2.0 + LLVM/Clang/LLD/LLDB 16.0.0 + MinGW-w64 10.0.0 (UCRT) - release 5
+* GCC 12.2.0 + LLVM/Clang/LLD/LLDB 16.0.0 + MinGW-w64 10.0.0 (MSVCRT) - release 5
+* GCC 11.2.0 + MinGW-w64 10.0.0 (MSVCRT) - release 1
+
 ## Use TsFile
 
 You can find examples on how to read and write data in `demo_read.cpp` and `demo_write.cpp` located under `./examples/cpp_examples`. There are also examples under `./examples/c_examples`on how to use a C-style API to read and write data in a C environment. You can run `bash build.sh` under `./examples` to generate an executable output under `./examples/build`.
diff --git a/cpp/examples/cpp_examples/demo_read.cpp b/cpp/examples/cpp_examples/demo_read.cpp
@@ -70,16 +70,16 @@ int demo_read() {
 
     std::cout << "begin to query expr" << std::endl;
     ASSERT(ret == 0);
-    storage::QueryDataSet *qds = nullptr;
+    storage::ResultSet *qds = nullptr;
     ret = reader.query(query_expr, qds);
 
     storage::RowRecord *record;
     std::cout << "begin to dump data from tsfile ---" << std::endl;
     int row_cout = 0;
     do {
-        record = qds->get_next();
-        if (record) {
+        if (qds->next()) {
             std::cout << "dump QDS :  " << record->get_timestamp() << ",";
+            record = qds->get_row_record();
             if (record) {
                 int size = record->get_fields()->size();
                 for (int i = 0; i < size; ++i) {

diff --git a/cpp/examples/cpp_examples/demo_write.cpp b/cpp/examples/cpp_examples/demo_write.cpp
@@ -66,16 +66,14 @@ int demo_write() {
     std::vector<MeasurementSchema> schema_vec[50];
     for (int i = 0; i < device_num; i++) {
         std::string device_name = "test_device" + std::to_string(i);
+        schema_vec[i].reserve(measurement_num);
         for (int j = 0; j < measurement_num; j++) {
             std::string measure_name = "measurement" + std::to_string(j);
-            schema_vec[i].push_back(
+            schema_vec[i].emplace_back(
                 MeasurementSchema(measure_name, common::TSDataType::INT32,
                                   common::TSEncoding::PLAIN,
                                   common::CompressionType::UNCOMPRESSED));
-            tsfile_writer_->register_timeseries(
-                device_name, measure_name, common::TSDataType::INT32,
-                common::TSEncoding::PLAIN,
-                common::CompressionType::UNCOMPRESSED);
+            tsfile_writer_->register_timeseries(device_name, schema_vec[i][j]);
         }
     }
 

diff --git a/cpp/pom.xml b/cpp/pom.xml
@@ -147,6 +147,40 @@
         </plugins>
     </build>
     <profiles>
+        <profile>
+            <id>linux-install-uuid-dev</id>
+            <activation>
+                <os>
+                    <family>unix</family>
+                    <name>Linux</name>
+                </os>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <version>1.1.1</version>
+                        <executions>
+                            <execution>
+                                <id>install-uuid-dev</id>
+                                <phase>validate</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                            </execution>
+                        </executions>
+                        <configuration>
+                            <executable>bash</executable>
+                            <arguments>
+                                <argument>-c</argument>
+                                <argument>sudo apt-get update &amp;&amp; sudo apt-get install -y uuid-dev</argument>
+                            </arguments>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
         <profile>
             <id>with-code-coverage</id>
             <properties>

diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt
@@ -20,6 +20,9 @@ message("Running in src diectory")
 if (${COV_ENABLED})
     add_compile_options(-fprofile-arcs -ftest-coverage)
 endif ()
+add_definitions(-DANTLR4CPP_STATIC)
+set(ANTLR4_WITH_STATIC_CRT OFF)
+add_subdirectory(parser)
 add_subdirectory(common)
 add_subdirectory(compress)
 add_subdirectory(cwrapper)
@@ -29,13 +32,22 @@ add_subdirectory(reader)
 add_subdirectory(utils)
 add_subdirectory(writer)
 
+set(SNAPPY_LIB_NAME "snappy")
+set(LZ4_LIB_NAME "LZ4")
+set(LZO_LIB_NAME "lzokay")
+target_link_libraries(parser_obj antlr4_static)
+target_link_libraries(compress_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} zlibstatic )
+target_link_libraries(common_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} zlibstatic )
+target_link_libraries(read_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} zlibstatic )
+target_link_libraries(write_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} zlibstatic )
+
 add_library(tsfile SHARED)
 if (${COV_ENABLED})
     message("Enable code cov...")
-    target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj -lgcov)
+    target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj -lgcov)
 else()
     message("Disable code cov...")
-    target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj)
+    target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj)
 endif()
 
 set(LIBTSFILE_PROJECT_VERSION ${TsFile_CPP_VERSION})
@@ -46,15 +58,6 @@ set_target_properties(tsfile PROPERTIES SOVERSION ${LIBTSFILE_SO_VERSION})
 set(LIBTSFILE_SDK_DIR ${LIBRARY_OUTPUT_PATH})
 install(TARGETS tsfile LIBRARY DESTINATION ${LIBTSFILE_SDK_DIR})
 
-set(SNAPPY_LIB_NAME "snappy")
-set(LZ4_LIB_NAME "LZ4")
-set(LZO_LIB_NAME "lzokay")
-set(ZLIB_LIB_NAME "z")
-
-target_link_libraries(compress_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
-target_link_libraries(common_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
-target_link_libraries(read_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
-target_link_libraries(write_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
 # set(CMAKE_PREFIX_PATH ../../third-party/lz4-dev/lib)
 # set(LZ4_LIB_DIR ../../third-party/lz4-dev/lib)
 # find_library(my_lz4_lib NAMES lz4 PATHS ${LZ4_LIB_DIR} NO_DEFAULT_PATH REQUIRED)

diff --git a/cpp/src/common/allocator/my_string.h b/cpp/src/common/allocator/my_string.h
@@ -147,6 +147,7 @@ struct String {
 
         return this->len_ < other.len_;
     }
+    std::string to_std_string() { return std::string(buf_, len_); }
 
 #ifndef NDEBUG
     friend std::ostream &operator<<(std::ostream &os, const String &s) {

diff --git a/cpp/src/common/constant/CMakeLists.txt b/cpp/src/common/constant/CMakeLists.txt
@@ -0,0 +1,18 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
diff --git a/cpp/src/common/constant/tsfile_constant.h b/cpp/src/common/constant/tsfile_constant.h
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <string>
+#include <regex>
+
+namespace storage
+{
+    static const std::string TSFILE_SUFFIX = ".tsfile";
+    static const std::string TSFILE_HOME = "TSFILE_HOME";
+    static const std::string TSFILE_CONF = "TSFILE_CONF";
+    static const std::string PATH_ROOT = "root";
+    static const std::string TMP_SUFFIX = "tmp";
+    static const std::string PATH_SEPARATOR = ".";
+    static const char PATH_SEPARATOR_CHAR = '.';
+    static const std::string PATH_SEPARATER_NO_REGEX = "\\.";
+    static const char DOUBLE_QUOTE = '"';
+    static const char BACK_QUOTE = '`';
+    static const std::string BACK_QUOTE_STRING = "`";
+    static const std::string DOUBLE_BACK_QUOTE_STRING = "``";
+
+    static const unsigned char TIME_COLUMN_MASK = 0x80;
+    static const unsigned char VALUE_COLUMN_MASK = 0x40;
+
+    static const std::string TIME_COLUMN_ID = "";
+
+    static const std::regex IDENTIFIER_PATTERN("([a-zA-Z0-9_\\u2E80-\\u9FFF]+)");
+    static const std::regex NODE_NAME_PATTERN("(\\*{0,2}[a-zA-Z0-9_\\u2E80-\\u9FFF]+\\*{0,2})");
+} // namespace storage
diff --git a/cpp/src/common/mutex/mutex.h b/cpp/src/common/mutex/mutex.h
@@ -43,6 +43,7 @@ class Mutex {
     void unlock() {
         int ret = pthread_mutex_unlock(&mutex_);
         ASSERT(ret == 0);
+        (void) ret;
     }
 
     bool try_lock() {

diff --git a/cpp/src/common/path.h b/cpp/src/common/path.h
@@ -19,6 +19,10 @@
 #ifndef COMMON_READ_COMMON_PATH_H
 #define COMMON_READ_COMMON_PATH_H
 
+#include "utils/errno_define.h"
+#include "parser/generated/PathParser.h"
+#include "parser/path_nodes_generator.h"
+
 #include <string>
 
 namespace storage {
@@ -35,6 +39,28 @@ struct Path {
         full_path_ = device + "." + measurement;
     }
 
+    Path(const std::string& path_sc, bool if_split = true) {
+        if (!path_sc.empty()) {
+            if (!if_split) {
+                full_path_ = path_sc;
+                device_ = path_sc; 
+            } else {
+                std::vector<std::string> nodes = PathNodesGenerator::invokeParser(path_sc);
+                if (nodes.size() > 0) {
+                    for (uint64_t i = 0; i + 1 < nodes.size(); i++) {
+                        device_ += nodes[i] + (i + 2 < nodes.size() ? "." : "");
+                    }
+                    measurement_ = nodes[nodes.size() - 1];
+                    full_path_ = device_ + "." + measurement_;
+                } else {
+                    full_path_ = path_sc;
+                    device_ = "";
+                    measurement_ = path_sc;
+                }
+            }
+        }
+    }
+
     bool operator==(const Path &path) {
         if (measurement_.compare(path.measurement_) == 0 &&
             device_.compare(path.device_) == 0) {

diff --git a/cpp/src/common/record.h b/cpp/src/common/record.h
@@ -24,7 +24,7 @@
 #include <vector>
 
 #include "common/db_common.h"
-
+#include "utils/errno_define.h"
 namespace storage {
 
 // TODO: use std::move
@@ -120,22 +120,24 @@ struct DataPoint {
 
 struct TsRecord {
     int64_t timestamp_;
-    std::string device_name_;
+    std::string device_id_;
     std::vector<DataPoint> points_;
 
-    TsRecord(const std::string &device_name) : device_name_(device_name) {}
+    TsRecord(const std::string &device_name) : device_id_(device_name) {}
 
     TsRecord(int64_t timestamp, const std::string &device_name,
              int32_t point_count_in_row = 0)
-        : timestamp_(timestamp), device_name_(device_name), points_() {
+        : timestamp_(timestamp), device_id_(device_name), points_() {
         if (point_count_in_row > 0) {
             points_.reserve(point_count_in_row);
         }
     }
 
-    void append_data_point(const DataPoint &point) {
-        // points_.emplace_back(point); C++11
-        points_.push_back(point);
+    template <typename T>
+    int add_point(const std::string &measurement_name, T val) {
+        int ret = common::E_OK;
+        points_.emplace_back(DataPoint(measurement_name, val));
+        return ret;
     }
 };
 

diff --git a/cpp/src/common/row_record.h b/cpp/src/common/row_record.h
@@ -86,9 +86,31 @@ struct Field {
         }
     }
 
+    template <typename T>
+    FORCE_INLINE T get_value() {
+        switch (type_) {
+            case common::TSDataType::BOOLEAN:
+                return value_.bval_;
+            case common::TSDataType::INT32:
+                return value_.ival_;
+            case common::TSDataType::INT64:
+                return value_.lval_;
+            case common::TSDataType::FLOAT:
+                return value_.fval_;
+            case common::TSDataType::DOUBLE:
+                return value_.dval_;
+            // case common::TSDataType::TEXT :
+            //     return value_.sval_;
+            default:
+                std::cout << "unknown data type" << std::endl;
+                break;
+        }
+        return -1;  // when data type is unknown
+    }
+
    public:
     common::TSDataType type_;
-
+    std::string column_name;
     union {
         bool bval_;
         int64_t lval_;
@@ -182,6 +204,8 @@ class RowRecord {
 
     FORCE_INLINE std::vector<Field *> *get_fields() { return fields_; }
 
+    FORCE_INLINE uint32_t get_col_num() { return col_num_; }
+
    private:
     int64_t time_;                  // time value
     uint32_t col_num_;              // measurement num

diff --git a/cpp/src/common/schema.h b/cpp/src/common/schema.h
@@ -50,6 +50,15 @@ struct MeasurementSchema {
           chunk_writer_(nullptr),
           value_chunk_writer_(nullptr) {}
 
+    MeasurementSchema(const std::string &measurement_name,
+                      common::TSDataType data_type)
+        : measurement_name_(measurement_name),
+          data_type_(data_type),
+          encoding_(get_default_encoding_for_type(data_type)),
+          compression_type_(common::LZ4),
+          chunk_writer_(nullptr),
+          value_chunk_writer_(nullptr) {}
+
     MeasurementSchema(const std::string &measurement_name,
                       common::TSDataType data_type, common::TSEncoding encoding,
                       common::CompressionType compression_type)