rewrite some Tsfile C++ interface

cpp/src/CMakeLists.txt

@@ -20,7 +20,7 @@ message("Running in src diectory")
 if (${COV_ENABLED})
     add_compile_options(-fprofile-arcs -ftest-coverage)
 endif ()
-add_subdirectory(Parser)
+add_subdirectory(parser)
 add_subdirectory(common)
 add_subdirectory(compress)
 add_subdirectory(cwrapper)

cpp/src/common/path.h

@@ -20,8 +20,8 @@
 #define COMMON_READ_COMMON_PATH_H
 
 #include "utils/errno_define.h"
-#include "parser/PathParser.h"
-#include "parser/PathNodesGenerator.h"
+#include "parser/generator/PathParser.h"
+#include "parser/path_nodes_generator.h"
 
 #include <string>
 
@@ -47,7 +47,7 @@ struct Path {
             } else {
                 std::vector<std::string> nodes = PathNodesGenerator::invokeParser(path_sc);
                 if (nodes.size() > 0) {
-                    for (int i = 0; i + 1 < nodes.size(); i++) {
+                    for (uint64_t i = 0; i + 1 < nodes.size(); i++) {
                         device_ += nodes[i] + (i + 2 < nodes.size() ? "." : "");
                     }
                     measurement_ = nodes[nodes.size() - 1];

cpp/src/common/tablet.cc

@@ -77,7 +77,7 @@ void Tablet::destroy() {
 
 int Tablet::add_timestamp(uint32_t row_index, int64_t timestamp) {
     ASSERT(timestamps_ != NULL);
-    if (UNLIKELY(row_index >= max_row_num_)) {
+    if (UNLIKELY(row_index >= static_cast<uint32_t>(max_row_num_))) {
         ASSERT(false);
         return E_OUT_OF_RANGE;
     }

cpp/src/common/tsfile_common.h

@@ -699,10 +699,11 @@ enum MetaIndexNodeType {
     LEAF_MEASUREMENT = 3,
     INVALID_META_NODE_TYPE = 4,
 };
-
+#ifndef NDEBUG
 static const char *meta_index_node_type_names[5] = {
     "INTERNAL_DEVICE", "LEAF_DEVICE", "INTERNAL_MEASUREMENT",
     "LEAF_MEASUREMENT", "INVALID_META_NODE_TYPE"};
+#endif
 
 struct MetaIndexNode {
     // TODO use vector to support binary search

cpp/src/file/CMakeLists.txt

@@ -16,7 +16,7 @@ KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations
 under the License.
 ]]
-message("running in src/fuke diectory")
+message("running in src/file diectory")
 
 message("CMAKE_CURRENT_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}")
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)

cpp/src/parser/CMakeLists.txt

@@ -16,7 +16,7 @@ KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations
 under the License.
 ]]
-message("Running in src/storage/tsfile/Parser directory")
+message("Running in src/parser directory")
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-aux_source_directory(. PARSER_SRC_LIST)
+file(GLOB_RECURSE PARSER_SRC_LIST "*.cpp")
 add_library(parser_obj OBJECT ${PARSER_SRC_LIST})

cpp/src/parser/PathNodesGenerator.cpp → cpp/src/parser/path_nodes_generator.cpp

@@ -19,30 +19,28 @@
 #include <string>
 #include <vector>
 
-#include "PathNodesGenerator.h"
+#include "path_nodes_generator.h"
 #include "utils/errno_define.h"
-#include "PathLexer.h"
-#include "PathParser.h"
-#include "PathParserError.h"
+#include "generator/PathLexer.h"
+#include "generator/PathParser.h"
+#include "path_parser_error.h"
 #include "path_visitor.h"
 
 namespace storage {
     std::vector<std::string> PathNodesGenerator::invokeParser(const std::string& path) {
         antlr4::ANTLRInputStream inputStream(path);
         PathLexer lexer(&inputStream);
         lexer.removeErrorListeners(); 
-        // lexer.addErrorListener(&PathParseError::getInstance());
+        lexer.addErrorListener(&PathParseError::getInstance());
         antlr4::CommonTokenStream tokens(&lexer);
         PathParser parser(&tokens);
         parser.removeErrorListeners(); 
-        // parser.addErrorListener(&PathParseError::getInstance());
+        parser.addErrorListener(&PathParseError::getInstance());
         parser.getInterpreter<antlr4::atn::ParserATNSimulator>()->setPredictionMode(antlr4::atn::PredictionMode::LL);
-
         /* if use SLL Mode to parse path, it will throw exception
             but c++ tsfile forbid throw exception, so we use LL Mode
             to parse path.
         */
-
         PathVisitor path_visitor;
         return path_visitor.visit(parser.path()).as<std::vector<std::string>>();
     }

cpp/src/parser/PathParserError.h → cpp/src/parser/path_parser_error.h

@@ -47,7 +47,6 @@ class PathParseError : public antlr4::BaseErrorListener {
                 expectedTokenNames.insert(vocabulary.getDisplayName(token));
             }
 
-            // 修改错误消息
             if (expectedTokenNames.count("ID") && expectedTokenNames.count("QUOTED_ID")) {
                 std::ostringstream expectedStr;
                 expectedStr << "{ID, QUOTED_ID";

cpp/src/parser/path_visitor.cpp

@@ -34,7 +34,7 @@ namespace storage
         std::vector<std::string> path;
         path.reserve(node_names.size() + 1);
         path.push_back(ctx->ROOT()->getText());
-        for (int i = 0; i < node_names.size(); i++) {
+        for (uint64_t i = 0; i < node_names.size(); i++) {
             path.push_back(parse_node_name(node_names[i]));
         }
         return path;
@@ -44,24 +44,23 @@ namespace storage
         std::vector<PathParser::NodeNameContext *> node_names = ctx->nodeName();
         std::vector<std::string> path;
         path.reserve(node_names.size());
-        for (int i = 0; i < node_names.size(); i++) {
-            path.push_back(parse_node_name(node_names[i]));
+        for (uint64_t i = 0; i < node_names.size(); i++) {
+            path.emplace_back(parse_node_name(node_names[i]));
         }
         return path;
     }
     std::string PathVisitor::parse_node_name(PathParser::NodeNameContext *ctx) {
-        std::string nodeName = ctx->getText();
-        if (starts_with(nodeName, BACK_QUOTE_STRING) && ends_with(nodeName, BACK_QUOTE_STRING)) {
-            std::string unWrapped = nodeName.substr(1, nodeName.length() - 1);
-
+        std::string node_name = ctx->getText();
+        if (starts_with(node_name, BACK_QUOTE_STRING) && ends_with(node_name, BACK_QUOTE_STRING)) {
+            std::string unWrapped = node_name.substr(1, node_name.length() - 2);
             if (is_real_number(unWrapped) || !std::regex_match(unWrapped, IDENTIFIER_PATTERN)) {
-                return nodeName;
+                return node_name;
             }
 
             return unWrapped;
         }
 
-        return nodeName;
+        return node_name;
     }
     bool PathVisitor::is_real_number(const std::string& str) {
         std::string s = str;
@@ -73,7 +72,11 @@ namespace storage
                 s = removeSign;
             }
         }
-        size_t index = std::find_if(s.begin(), s.end(), [](const char& c) { return c != '0'; }) - s.begin();
+        size_t index = 0;
+        auto it = std::find_if(s.begin(), s.end(), [](const char& c) { return c != '0'; });
+        if (it != s.end()) {
+            index = it - s.begin();
+        }
 
         if (index > 0 && (s[index] == 'e' || s[index] == 'E')) {
             return is_creatable(s.substr(index - 1));

cpp/src/parser/path_visitor.h

@@ -20,8 +20,8 @@
 #ifndef PATH_VISITOR_H
 #define PATH_VISITOR_H
 
-#include "PathParserBaseVisitor.h"
-#include "PathParser.h"
+#include "generator/PathParserBaseVisitor.h"
+#include "generator/PathParser.h"
 #include "common/constant/tsfile_constant.h"
 
 namespace storage

cpp/src/reader/tsfile_reader.cc

@@ -122,6 +122,7 @@ int TsFileReader::get_all_devices(std::vector<std::string> &device_ids, MetaInde
                 } else {
                     ret = get_all_devices(device_ids, top_node, pa);
                 }
+                top_node->destroy();
             }
         }
     }

cpp/test/parser/path_name_test.cc

@@ -18,8 +18,8 @@
  */
 #include <gtest/gtest.h>
 
-#include "parser/PathParser.h"
-#include "parser/PathLexer.h"
+#include "parser/generator/PathParser.h"
+#include "parser/generator/PathLexer.h"
 #include "common/path.h"
 
 namespace storage {
@@ -41,11 +41,151 @@ TEST_F(PathNameTest, TestPathLexer) {
     EXPECT_EQ(actualTokens, expectedTokens);
 }
 
-TEST_F(PathNameTest, TestPathNodeGenerator) {
-    const std::string path_str = "root.sg1.`.d1`.s1";
-    Path path(path_str, true);
-    EXPECT_EQ(path.device_, "root.sg1.`.d1`");
-    EXPECT_EQ(path.measurement_, "s1");
+TEST_F(PathNameTest, TestLegalPath) {
+    // empty path
+    Path a("", true);
+    EXPECT_EQ("", a.device_);
+    EXPECT_EQ("", a.measurement_);
+
+    // empty device
+    Path b("s1", true);
+    EXPECT_EQ("s1", b.measurement_);
+    EXPECT_EQ("", b.device_);
+
+    // normal node
+    Path c("root.sg.a", true);
+    EXPECT_EQ("root.sg", c.device_);
+    EXPECT_EQ("a", c.measurement_);
+
+    // quoted node
+    Path d("root.sg.`a.b`", true);
+    EXPECT_EQ("root.sg", d.device_);
+    EXPECT_EQ("`a.b`", d.measurement_);
+
+    Path e("root.sg.`a.``b`", true);
+    EXPECT_EQ("root.sg", e.device_);
+    EXPECT_EQ("`a.``b`", e.measurement_);
+
+    Path f("root.`sg\"`.`a.``b`", true);
+    EXPECT_EQ("root.`sg\"`", f.device_);
+    EXPECT_EQ("`a.``b`", f.measurement_);
+
+    Path g("root.sg.`a.b\\\\`", true);
+    EXPECT_EQ("root.sg", g.device_);
+    EXPECT_EQ("`a.b\\\\`", g.measurement_);
+
+    // quoted node of digits
+    Path h("root.sg.`111`", true);
+    EXPECT_EQ("root.sg", h.device_);
+    EXPECT_EQ("`111`", h.measurement_);
+
+    // quoted node of key word
+    Path i("root.sg.`select`", true);
+    EXPECT_EQ("root.sg", i.device_);
+    EXPECT_EQ("select", i.measurement_);
+
+    // wildcard
+    Path j("root.sg.`a*b`", true);
+    EXPECT_EQ("root.sg", j.device_);
+    EXPECT_EQ("`a*b`", j.measurement_);
+
+    Path k("root.sg.*", true);
+    EXPECT_EQ("root.sg", k.device_);
+    EXPECT_EQ("*", k.measurement_);
+
+    Path l("root.sg.**", true);
+    EXPECT_EQ("root.sg", l.device_);
+    EXPECT_EQ("**", l.measurement_);
+
+    // raw key word
+    Path m("root.sg.select", true);
+    EXPECT_EQ("root.sg", m.device_);
+    EXPECT_EQ("select", m.measurement_);
+
+    Path n("root.sg.device", true);
+    EXPECT_EQ("root.sg", n.device_);
+    EXPECT_EQ("device", n.measurement_);
+
+    Path o("root.sg.drop_trigger", true);
+    EXPECT_EQ("root.sg", o.device_);
+    EXPECT_EQ("drop_trigger", o.measurement_);
+
+    Path p("root.sg.and", true);
+    EXPECT_EQ("root.sg", p.device_);
+    EXPECT_EQ("and", p.measurement_);
+
+    p = Path("root.sg.or", true);
+    EXPECT_EQ("root.sg", p.device_);
+    EXPECT_EQ("or", p.measurement_);
+
+    p = Path("root.sg.not", true);
+    EXPECT_EQ("root.sg", p.device_);
+    EXPECT_EQ("not", p.measurement_);
+
+    p = Path("root.sg.null", true);
+    EXPECT_EQ("root.sg", p.device_);
+    EXPECT_EQ("null", p.measurement_);
+
+    p = Path("root.sg.contains", true);
+    EXPECT_EQ("root.sg", p.device_);
+    EXPECT_EQ("contains", p.measurement_);
+
+    p = Path("root.sg.`0000`", true);
+    EXPECT_EQ("root.sg", p.device_);
+    EXPECT_EQ("`0000`", p.measurement_);
+
+    p = Path("root.sg.`0e38`", true);
+    EXPECT_EQ("root.sg", p.device_);
+    EXPECT_EQ("`0e38`", p.measurement_);
+
+    p = Path("root.sg.`00.12`", true);
+    EXPECT_EQ("root.sg", p.device_);
+    EXPECT_EQ("`00.12`", p.measurement_);
+}
+
+TEST_F(PathNameTest, TestIllegalPathName) {
+    EXPECT_THROW ({
+      Path("root.sg`", true);
+    } , std::runtime_error);
+
+    EXPECT_THROW ({
+      Path("root.sg\na", true);
+    }, std::runtime_error);
+
+    EXPECT_THROW ({
+      Path("root.select`", true);
+    } , std::runtime_error);
+
+    EXPECT_THROW ({
+      // pure digits
+      Path("root.111", true);
+    } , std::runtime_error);
+
+    EXPECT_THROW ({
+      // single ` in quoted node
+      Path("root.`a``", true);
+    } , std::runtime_error);
+
+    EXPECT_THROW ({
+      // single ` in quoted node
+      Path("root.``a`", true);
+    } , std::runtime_error);
+
+    EXPECT_THROW ({
+      Path("root.a*%", true);
+    } , std::runtime_error);
+
+    EXPECT_THROW ({
+      Path("root.a*b", true);
+    } , std::runtime_error);
+
+    EXPECT_THROW ({
+      Path("root.0e38", true);
+    } , std::runtime_error);
+
+    EXPECT_THROW ({
+      Path("root.0000", true);
+    }, std::runtime_error);
 }
 
 

cpp/third_party/antlr4-cpp-runtime-4/runtime/src/atn/ATN.cpp

@@ -158,7 +158,7 @@ misc::IntervalSet ATN::getExpectedTokens(size_t stateNumber, RuleContext *contex
   }
 
   if (following.contains(Token::EPSILON)) {
-    expected.add(Token::EOF);
+    expected.add(static_cast<int>(Token::EOF));
   }
 
   return expected;

cpp/third_party/antlr4-cpp-runtime-4/runtime/src/atn/LL1Analyzer.cpp

@@ -82,20 +82,20 @@ void LL1Analyzer::_LOOK(ATNState *s, ATNState *stopState, Ref<PredictionContext>
   // ml: s can never be null, hence no need to check if stopState is != null.
   if (s == stopState) {
     if (ctx == nullptr) {
-      look.add(Token::EPSILON);
+      look.add(static_cast<int>(Token::EPSILON));
       return;
     } else if (ctx->isEmpty() && addEOF) {
-      look.add(Token::EOF);
+      look.add(static_cast<int>(Token::EOF));
       return;
     }
   }
 
   if (s->getStateType() == ATNState::RULE_STOP) {
     if (ctx == nullptr) {
-      look.add(Token::EPSILON);
+      look.add(static_cast<int>(Token::EPSILON));
       return;
     } else if (ctx->isEmpty() && addEOF) {
-      look.add(Token::EOF);
+      look.add(static_cast<int>(Token::EOF));
       return;
     }
 

cpp/third_party/antlr4-cpp-runtime-4/runtime/src/support/CPPUtils.cpp

@@ -48,6 +48,9 @@ namespace antlrcpp {
           if (escapeSpaces) {
             result += "\u00B7";
             break;
+          } else {
+            result += c;
+            break;
           }
           // else fall through
 #ifndef _MSC_VER