Add parser for path and fix for get device name

apache · Dec 10, 2024 · bd55cda · bd55cda
1 parent b10f548
commit bd55cda
Show file tree

Hide file tree

Showing 401 changed files with 47,039 additions and 47 deletions.
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -76,6 +76,8 @@ set(PROJECT_INCLUDE_DIR ${PROJECT_INCLUDE_DIR}
 set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
 include_directories(${PROJECT_INCLUDE_DIR})
 
+include_directories(${PROJECT_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src)
+
 add_subdirectory(third_party)
 
 add_subdirectory(src)

diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt
@@ -20,6 +20,7 @@ message("Running in src diectory")
 if (${COV_ENABLED})
     add_compile_options(-fprofile-arcs -ftest-coverage)
 endif ()
+add_subdirectory(Parser)
 add_subdirectory(common)
 add_subdirectory(compress)
 add_subdirectory(cwrapper)
@@ -32,10 +33,10 @@ add_subdirectory(writer)
 add_library(tsfile SHARED)
 if (${COV_ENABLED})
     message("Enable code cov...")
-    target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj -lgcov)
+    target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj -lgcov)
 else()
     message("Disable code cov...")
-    target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj)
+    target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj)
 endif()
 
 set(LIBTSFILE_PROJECT_VERSION ${TsFile_CPP_VERSION})
@@ -50,9 +51,10 @@ set(SNAPPY_LIB_NAME "snappy")
 set(LZ4_LIB_NAME "LZ4")
 set(LZO_LIB_NAME "lzokay")
 set(ZLIB_LIB_NAME "z")
-
+set(ANTLR4_LIB_PATH ${CMAKE_BINARY_DIR}/lib/libantlr4-runtime${CMAKE_SHARED_LIBRARY_SUFFIX})
+target_link_libraries(parser_obj ${ANTLR4_LIB_PATH})
 target_link_libraries(compress_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
-target_link_libraries(common_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
+target_link_libraries(common_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME} ${ANTLR4_LIB_PATH})
 target_link_libraries(read_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
 target_link_libraries(write_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
 # set(CMAKE_PREFIX_PATH ../../third-party/lz4-dev/lib)

diff --git a/cpp/src/common/constant/CMakeLists.txt b/cpp/src/common/constant/CMakeLists.txt
@@ -0,0 +1,18 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
diff --git a/cpp/src/common/constant/tsfile_constant.h b/cpp/src/common/constant/tsfile_constant.h
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <string>
+#include <regex>
+
+namespace storage
+{
+    static const std::string TSFILE_SUFFIX = ".tsfile";
+    static const std::string TSFILE_HOME = "TSFILE_HOME";
+    static const std::string TSFILE_CONF = "TSFILE_CONF";
+    static const std::string PATH_ROOT = "root";
+    static const std::string TMP_SUFFIX = "tmp";
+    static const std::string PATH_SEPARATOR = ".";
+    static const char PATH_SEPARATOR_CHAR = '.';
+    static const std::string PATH_SEPARATER_NO_REGEX = "\\.";
+    static const char DOUBLE_QUOTE = '"';
+    static const char BACK_QUOTE = '`';
+    static const std::string BACK_QUOTE_STRING = "`";
+    static const std::string DOUBLE_BACK_QUOTE_STRING = "``";
+
+    static const unsigned char TIME_COLUMN_MASK = 0x80;
+    static const unsigned char VALUE_COLUMN_MASK = 0x40;
+
+    static const std::string TIME_COLUMN_ID = "";
+
+    static const std::regex IDENTIFIER_PATTERN("([a-zA-Z0-9_\\u2E80-\\u9FFF]+)");
+    static const std::regex NODE_NAME_PATTERN("(\\*{0,2}[a-zA-Z0-9_\\u2E80-\\u9FFF]+\\*{0,2})");
+} // namespace storage
diff --git a/cpp/src/common/mutex/mutex.h b/cpp/src/common/mutex/mutex.h
@@ -43,6 +43,7 @@ class Mutex {
     void unlock() {
         int ret = pthread_mutex_unlock(&mutex_);
         ASSERT(ret == 0);
+        (void) ret;
     }
 
     bool try_lock() {

diff --git a/cpp/src/common/path.h b/cpp/src/common/path.h
@@ -19,6 +19,10 @@
 #ifndef COMMON_READ_COMMON_PATH_H
 #define COMMON_READ_COMMON_PATH_H
 
+#include "utils/errno_define.h"
+#include "parser/PathParser.h"
+#include "parser/PathNodesGenerator.h"
+
 #include <string>
 
 namespace storage {
@@ -35,6 +39,28 @@ struct Path {
         full_path_ = device + "." + measurement;
     }
 
+    Path(const std::string& path_sc, bool if_split = true) {
+        if (!path_sc.empty()) {
+            if (!if_split) {
+                full_path_ = path_sc;
+                device_ = path_sc; 
+            } else {
+                std::vector<std::string> nodes = PathNodesGenerator::invokeParser(path_sc);
+                if (nodes.size() > 0) {
+                    for (int i = 0; i + 1 < nodes.size(); i++) {
+                        device_ += nodes[i] + (i + 2 < nodes.size() ? "." : "");
+                    }
+                    measurement_ = nodes[nodes.size() - 1];
+                    full_path_ = device_ + "." + measurement_;
+                } else {
+                    full_path_ = path_sc;
+                    device_ = "";
+                    measurement_ = path_sc;
+                }
+            }
+        }
+    }
+
     bool operator==(const Path &path) {
         if (measurement_.compare(path.measurement_) == 0 &&
             device_.compare(path.device_) == 0) {

diff --git a/cpp/src/common/tablet.h b/cpp/src/common/tablet.h
@@ -41,7 +41,7 @@ class Tablet {
    public:
     Tablet(const std::string &device_id,
            std::shared_ptr<std::vector<MeasurementSchema>> schema_vec,
-           uint32_t max_rows = DEFAULT_MAX_ROWS)
+           int max_rows = DEFAULT_MAX_ROWS)
         : max_row_num_(max_rows),
           device_id_(device_id),
           schema_vec_(schema_vec),
@@ -60,7 +60,7 @@ class Tablet {
     Tablet(const std::string &device_id,
            const std::vector<std::string> *measurement_list,
            const std::vector<common::TSDataType> *data_type_list,
-           uint32_t max_row_num = DEFAULT_MAX_ROWS)
+           int max_row_num = DEFAULT_MAX_ROWS)
         : max_row_num_(max_row_num),
           device_id_(device_id),
           timestamps_(NULL),
@@ -108,7 +108,7 @@ class Tablet {
     typedef std::map<std::string, int>::iterator SchemaMapIterator;
 
    private:
-    uint32_t max_row_num_;
+    int max_row_num_;
     std::string device_id_;
     std::shared_ptr<std::vector<MeasurementSchema>> schema_vec_;
     std::map<std::string, int> schema_map_;

diff --git a/cpp/src/cwrapper/TsFile-cwrapper.cc b/cpp/src/cwrapper/TsFile-cwrapper.cc
@@ -697,8 +697,6 @@ DataResult* ts_next(QueryDataRet data, int expect_line_count) {
     for (int i = 0; i < expect_line_count; i++) {
         if (!qds->next()) {
             break;
-            std::cout << "no more record now"
-                      << "i = " << i << std::endl;
         }
         record = qds->get_row_record();
         int column_num = record->get_fields()->size();

diff --git a/cpp/src/file/open_file.cc b/cpp/src/file/open_file.cc
@@ -25,6 +25,7 @@ namespace storage {
 
 int OpenFile::init() {
     void *buf = mem_alloc(sizeof(TsTimeRangeMap), MOD_OPEN_FILE_OBJ);
+    fd_ = -1;
     if (IS_NULL(buf)) {
         return E_OOM;
     }

diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc
@@ -525,7 +525,6 @@ int TsFileIOReader::do_load_all_timeseries_index(
     for (const auto &index_node_entry : index_node_entry_list) {
         int64_t start_offset = index_node_entry.first->offset_,
                 end_offset = index_node_entry.second;
-        std::cout << index_node_entry.first->name_ << std::endl;
         const std::string target_measurement_name(
             index_node_entry.first->name_.to_std_string());
         ITimeseriesIndex *ts_idx;
@@ -556,7 +555,7 @@ int TsFileIOReader::get_all_leaf(
             }
         }
     } else {
-        // reader next level index node
+        // read next level index node
         for (size_t i = 0; i < index_node->children_.size(); i++) {
             int64_t end_offset = index_node->end_offset_;
             if (i + 1 < index_node->children_.size()) {

diff --git a/cpp/src/file/tsfile_io_writer.cc b/cpp/src/file/tsfile_io_writer.cc
@@ -648,7 +648,6 @@ int TsFileIOWriter::generate_root(SimpleList<MetaIndexNode *> *node_queue,
     if (RET_FAIL(
             alloc_and_init_meta_index_node(wmm, cur_index_node, node_type))) {
     }
-    uint32_t from_size = from->size();
     while (IS_SUCC(ret)) {
         to->clear();
         SimpleList<MetaIndexNode *>::Iterator from_iter;
@@ -700,7 +699,7 @@ int TsFileIOWriter::generate_root(SimpleList<MetaIndexNode *> *node_queue,
             }
         }
         if (IS_SUCC(ret)) {
-            ASSERT(from_size > to->size());
+            ASSERT(from->size() > to->size());
             if (to->size() == 1) {
                 root_node = to->front();
                 break;

diff --git a/cpp/src/parser/CMakeLists.txt b/cpp/src/parser/CMakeLists.txt
@@ -0,0 +1,22 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+message("Running in src/storage/tsfile/Parser directory")
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+aux_source_directory(. PARSER_SRC_LIST)
+add_library(parser_obj OBJECT ${PARSER_SRC_LIST})