From 11433e1637a5eb428009b297fe7800f3ae77acbb Mon Sep 17 00:00:00 2001 From: Orlov Pavel Date: Wed, 4 Sep 2024 17:57:00 +0300 Subject: [PATCH] Migrating from antlr3 to antlr4 (#2977) Co-authored-by: root --- .gitignore | 1 + .../yql/parser/lexer_common/ut/hints_ut.cpp | 19 +- .../yql/parser/proto_ast/gen/jsonpath/ya.make | 5 +- .../yql/parser/proto_ast/gen/v0/ya.make | 4 + .../yql/parser/proto_ast/gen/v1/ya.make | 4 + .../yql/parser/proto_ast/gen/v1_ansi/ya.make | 4 + .../gen/v1_ansi_antlr4/epilogue.cmake | 10 + .../proto_ast/gen/v1_ansi_antlr4/ya.make | 52 + .../proto_ast/gen/v1_antlr4/epilogue.cmake | 9 + .../parser/proto_ast/gen/v1_antlr4/ya.make | 52 + .../proto_ast/gen/v1_proto_antlr4/ya.make.gen | 38 + .../gen/v1_proto_split_antlr4/epilogue.cmake | 9 + .../gen/v1_proto_split_antlr4/update.sh | 7 + .../gen/v1_proto_split_antlr4/ya.make | 87 + ydb/library/yql/parser/proto_ast/gen/ya.make | 2 + .../v4/tool/templates/codegen/Cpp/Cpp.stg.in | 1425 ++++ .../tool/templates/codegen/Cpp/Files.stg.in | 347 + .../tool/templates/codegen/Java/Java.stg.in | 459 ++ .../v4/tool/templates/codegen/ya.make.incl | 13 + .../yql/parser/proto_ast/proto_ast.cpp | 11 + ydb/library/yql/parser/proto_ast/proto_ast.h | 154 +- ydb/library/yql/parser/proto_ast/ya.make | 1 + .../yql/sql/settings/translation_settings.cpp | 3 + .../yql/sql/settings/translation_settings.h | 1 + ydb/library/yql/sql/sql.cpp | 2 +- ydb/library/yql/sql/v1/SQLv1Antlr4.g.in | 2091 +++++ ydb/library/yql/sql/v1/context.h | 9 + ydb/library/yql/sql/v1/format/sql_format.cpp | 9 +- .../yql/sql/v1/format/sql_format_ut.cpp | 1621 +--- ydb/library/yql/sql/v1/format/sql_format_ut.h | 1618 ++++ .../sql/v1/format/sql_format_ut_antlr4.cpp | 52 + .../yql/sql/v1/format/ut_antlr4/ya.make | 7 + ydb/library/yql/sql/v1/format/ya.make | 2 + ydb/library/yql/sql/v1/lexer/lexer.cpp | 22 +- ydb/library/yql/sql/v1/lexer/lexer.h | 2 +- ydb/library/yql/sql/v1/lexer/ya.make | 2 + .../yql/sql/v1/proto_parser/proto_parser.cpp | 29 +- .../yql/sql/v1/proto_parser/proto_parser.h | 4 +- ydb/library/yql/sql/v1/proto_parser/ya.make | 2 + ydb/library/yql/sql/v1/sql.cpp | 8 +- ydb/library/yql/sql/v1/sql_expression.cpp | 126 +- ydb/library/yql/sql/v1/sql_query.cpp | 62 +- ydb/library/yql/sql/v1/sql_query.h | 10 +- ydb/library/yql/sql/v1/sql_translation.cpp | 46 +- ydb/library/yql/sql/v1/sql_ut.cpp | 2 +- ydb/library/yql/sql/v1/sql_ut.h | 4 +- ydb/library/yql/sql/v1/sql_ut_antlr4.cpp | 7203 +++++++++++++++++ ydb/library/yql/sql/v1/sql_ut_antlr4.h | 226 + ydb/library/yql/sql/v1/ut_antlr4/ya.make | 20 + ydb/library/yql/sql/v1/ya.make | 3 + 50 files changed, 14126 insertions(+), 1773 deletions(-) create mode 100644 ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/epilogue.cmake create mode 100644 ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/ya.make create mode 100644 ydb/library/yql/parser/proto_ast/gen/v1_antlr4/epilogue.cmake create mode 100644 ydb/library/yql/parser/proto_ast/gen/v1_antlr4/ya.make create mode 100644 ydb/library/yql/parser/proto_ast/gen/v1_proto_antlr4/ya.make.gen create mode 100644 ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/epilogue.cmake create mode 100755 ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/update.sh create mode 100644 ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/ya.make create mode 100644 ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg.in create mode 100644 ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg.in create mode 100644 ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Java/Java.stg.in create mode 100644 ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/ya.make.incl create mode 100644 ydb/library/yql/sql/v1/SQLv1Antlr4.g.in create mode 100644 ydb/library/yql/sql/v1/format/sql_format_ut.h create mode 100644 ydb/library/yql/sql/v1/format/sql_format_ut_antlr4.cpp create mode 100644 ydb/library/yql/sql/v1/format/ut_antlr4/ya.make create mode 100644 ydb/library/yql/sql/v1/sql_ut_antlr4.cpp create mode 100644 ydb/library/yql/sql/v1/sql_ut_antlr4.h create mode 100644 ydb/library/yql/sql/v1/ut_antlr4/ya.make diff --git a/.gitignore b/.gitignore index 87b7b54d294d..13059c619843 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ __pycache__/ .idea/ .vscode/ .clangd +.antlr/ # KDevelop IDE *.kdev4 diff --git a/ydb/library/yql/parser/lexer_common/ut/hints_ut.cpp b/ydb/library/yql/parser/lexer_common/ut/hints_ut.cpp index 2567bc1c7d33..1ffd5377c79d 100644 --- a/ydb/library/yql/parser/lexer_common/ut/hints_ut.cpp +++ b/ydb/library/yql/parser/lexer_common/ut/hints_ut.cpp @@ -8,9 +8,9 @@ using namespace NSQLTranslation; using namespace NSQLTranslationV1; -TSQLHints CollectHints(const TString& query) { +TSQLHints CollectHints(const TString& query, bool antlr4Parser) { bool ansi = false; - auto lexer = MakeLexer(ansi); + auto lexer = MakeLexer(ansi, antlr4Parser); UNIT_ASSERT(lexer); TSQLHints result; NYql::TIssues issues; @@ -27,7 +27,20 @@ TString SerializeHints(const TVector& hints) { Y_UNIT_TEST_SUITE(TLexerHintsTests) { Y_UNIT_TEST(Basic) { TString query = "/*+ some() */ SELECT /*+ foo(one) */ --+ bar(two)"; - auto hintsWithPos = CollectHints(query); + auto hintsWithPos = CollectHints(query, false); + UNIT_ASSERT(hintsWithPos.size() == 1); + NYql::TPosition pos = hintsWithPos.begin()->first; + TVector hints = hintsWithPos.begin()->second; + + UNIT_ASSERT_EQUAL(pos.Row, 1); + UNIT_ASSERT_EQUAL(pos.Column, 15); + + TStringBuf expected = R"raw("foo":{"one"},"bar":{"two"})raw"; + UNIT_ASSERT_NO_DIFF(SerializeHints(hints), expected); + } + Y_UNIT_TEST(Antlr4) { + TString query = "/*+ some() */ SELECT /*+ foo(one) */ --+ bar(two)"; + auto hintsWithPos = CollectHints(query, true); UNIT_ASSERT(hintsWithPos.size() == 1); NYql::TPosition pos = hintsWithPos.begin()->first; TVector hints = hintsWithPos.begin()->second; diff --git a/ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make b/ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make index b050069fbc9f..557316988274 100644 --- a/ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/jsonpath/ya.make @@ -10,7 +10,6 @@ IF (CPP_PROTO) SET(PROTOBUF_SUFFIX_PATH .pb.h) SET(LEXER_PARSER_NAMESPACE NALP) - CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg) CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/protobuf/protobuf.stg.in ${antlr_templates}/protobuf/protobuf.stg) @@ -28,6 +27,10 @@ IF (CPP_PROTO) NO_COMPILER_WARNINGS() + ADDINCL( + GLOBAL contrib/libs/antlr4_cpp_runtime/src + ) + INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl) RUN_ANTLR( diff --git a/ydb/library/yql/parser/proto_ast/gen/v0/ya.make b/ydb/library/yql/parser/proto_ast/gen/v0/ya.make index 2037ad3f8645..15c84a512075 100644 --- a/ydb/library/yql/parser/proto_ast/gen/v0/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/v0/ya.make @@ -20,6 +20,10 @@ NO_COMPILER_WARNINGS() INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl) +ADDINCL( + GLOBAL contrib/libs/antlr4_cpp_runtime/src +) + RUN_ANTLR( ${sql_grammar} -lib . diff --git a/ydb/library/yql/parser/proto_ast/gen/v1/ya.make b/ydb/library/yql/parser/proto_ast/gen/v1/ya.make index f5487a56a184..c3586e6050e6 100644 --- a/ydb/library/yql/parser/proto_ast/gen/v1/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/v1/ya.make @@ -28,6 +28,10 @@ ENDIF() NO_COMPILER_WARNINGS() +ADDINCL( + GLOBAL contrib/libs/antlr4_cpp_runtime/src +) + INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl) RUN_ANTLR( diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make b/ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make index ebd0e8b70dc2..bd4614bc9475 100644 --- a/ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/v1_ansi/ya.make @@ -28,6 +28,10 @@ ENDIF() NO_COMPILER_WARNINGS() +ADDINCL( + GLOBAL contrib/libs/antlr4_cpp_runtime/src +) + INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl) RUN_ANTLR( diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/epilogue.cmake b/ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/epilogue.cmake new file mode 100644 index 000000000000..2559073edd72 --- /dev/null +++ b/ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/epilogue.cmake @@ -0,0 +1,10 @@ +set(GRAMMAR_STRING_CORE_SINGLE "~([']) | (QUOTE_SINGLE QUOTE_SINGLE)") +set(GRAMMAR_STRING_CORE_DOUBLE "~([\"]) | (QUOTE_DOUBLE QUOTE_DOUBLE)") +set(GRAMMAR_MULTILINE_COMMENT_CORE "MULTILINE_COMMENT | .") + +configure_file( + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in + ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4.g +) + + diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/ya.make b/ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/ya.make new file mode 100644 index 000000000000..c53f624301e8 --- /dev/null +++ b/ydb/library/yql/parser/proto_ast/gen/v1_ansi_antlr4/ya.make @@ -0,0 +1,52 @@ +LIBRARY() + +PEERDIR ( + ydb/library/yql/parser/proto_ast/gen/v1_proto_split +) + +SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) +SET(antlr_templates ${antlr_output}/org/antlr/v4/tool/templates/codegen) +SET(sql_grammar ${antlr_output}/SQLv1Antlr4.g) + +SET(ANTLR_PACKAGE_NAME NSQLv1Generated) +SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto_split) +SET(PROTOBUF_SUFFIX_PATH .pb.main.h) + +SET(LEXER_PARSER_NAMESPACE NALPAnsiAntlr4) + +SET(GRAMMAR_STRING_CORE_SINGLE "\"~([']) | (QUOTE_SINGLE QUOTE_SINGLE)\"") +SET(GRAMMAR_STRING_CORE_DOUBLE "\"~([#DOUBLE_QUOTE#]) | (QUOTE_DOUBLE QUOTE_DOUBLE)\"") +SET(GRAMMAR_MULTILINE_COMMENT_CORE "\"MULTILINE_COMMENT | .\"") + +CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg) +CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg.in ${antlr_templates}/Cpp/Files.stg) + +IF(EXPORT_CMAKE) + MANUAL_GENERATION(${sql_grammar}) +ELSE() + CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in ${sql_grammar}) +ENDIF() + +NO_COMPILER_WARNINGS() + +ADDINCL( + GLOBAL contrib/libs/antlr4_cpp_runtime/src +) + +INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/ya.make.incl) + +RUN_ANTLR4( + ${sql_grammar} + -no-listener + -package NALPAnsiAntlr4 + -lib . + -o ${antlr_output} + IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg ${antlr_templates}/Cpp/Files.stg + OUT SQLv1Antlr4Parser.cpp SQLv1Antlr4Lexer.cpp SQLv1Antlr4Parser.h SQLv1Antlr4Lexer.h + OUTPUT_INCLUDES + ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.main.h + ${STG_INCLUDES} + CWD ${antlr_output} +) + +END() diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_antlr4/epilogue.cmake b/ydb/library/yql/parser/proto_ast/gen/v1_antlr4/epilogue.cmake new file mode 100644 index 000000000000..4998610012eb --- /dev/null +++ b/ydb/library/yql/parser/proto_ast/gen/v1_antlr4/epilogue.cmake @@ -0,0 +1,9 @@ +set(GRAMMAR_STRING_CORE_SINGLE "~(['\\]) | (BACKSLASH .)") +set(GRAMMAR_STRING_CORE_DOUBLE "~([\"\\]) | (BACKSLASH .)") +set(GRAMMAR_MULTILINE_COMMENT_CORE ".") +configure_file( + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in + ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4.g +) + + diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_antlr4/ya.make b/ydb/library/yql/parser/proto_ast/gen/v1_antlr4/ya.make new file mode 100644 index 000000000000..016150d2cedb --- /dev/null +++ b/ydb/library/yql/parser/proto_ast/gen/v1_antlr4/ya.make @@ -0,0 +1,52 @@ +LIBRARY() + +PEERDIR ( + ydb/library/yql/parser/proto_ast/gen/v1_proto_split +) + +SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) +SET(antlr_templates ${antlr_output}/org/antlr/v4/tool/templates/codegen) +SET(sql_grammar ${antlr_output}/SQLv1Antlr4.g) + +SET(ANTLR_PACKAGE_NAME NSQLv1Generated) +SET(PROTOBUF_HEADER_PATH ydb/library/yql/parser/proto_ast/gen/v1_proto_split) +SET(PROTOBUF_SUFFIX_PATH .pb.main.h) + +SET(LEXER_PARSER_NAMESPACE NALPDefaultAntlr4) + +SET(GRAMMAR_STRING_CORE_SINGLE "\"~(['#BACKSLASH#]) | (BACKSLASH .)\"") +SET(GRAMMAR_STRING_CORE_DOUBLE "\"~([#DOUBLE_QUOTE##BACKSLASH#]) | (BACKSLASH .)\"") +SET(GRAMMAR_MULTILINE_COMMENT_CORE "\".\"") + +CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg.in ${antlr_templates}/Cpp/Cpp.stg) +CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg.in ${antlr_templates}/Cpp/Files.stg) + +IF(EXPORT_CMAKE) + MANUAL_GENERATION(${sql_grammar}) +ELSE() + CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in ${sql_grammar}) +ENDIF() + +NO_COMPILER_WARNINGS() + +ADDINCL( + GLOBAL contrib/libs/antlr4_cpp_runtime/src +) + +INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/codegen/templates/ya.make.incl) + +RUN_ANTLR4( + ${sql_grammar} + -no-listener + -package NALPDefaultAntlr4 + -lib . + -o ${antlr_output} + IN ${sql_grammar} ${antlr_templates}/Cpp/Cpp.stg ${antlr_templates}/Cpp/Files.stg + OUT SQLv1Antlr4Parser.cpp SQLv1Antlr4Lexer.cpp SQLv1Antlr4Parser.h SQLv1Antlr4Lexer.h + OUTPUT_INCLUDES + ${PROTOBUF_HEADER_PATH}/SQLv1Parser.pb.main.h + ${STG_INCLUDES} + CWD ${antlr_output} +) + +END() diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_proto_antlr4/ya.make.gen b/ydb/library/yql/parser/proto_ast/gen/v1_proto_antlr4/ya.make.gen new file mode 100644 index 000000000000..48df4c276040 --- /dev/null +++ b/ydb/library/yql/parser/proto_ast/gen/v1_proto_antlr4/ya.make.gen @@ -0,0 +1,38 @@ +PROTO_LIBRARY() + +IF (GEN_PROTO) + SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) + SET(antlr_templates ${antlr_output}/org/antlr/v4/tool/templates/codegen) + SET(sql_grammar ${antlr_output}/SQLv1Antlr4.g) + + SET(ANTLR_PACKAGE_NAME NSQLv1Generated) + + SET(GRAMMAR_STRING_CORE_SINGLE "\"~(['#BACKSLASH#]) | (BACKSLASH .)\"") + SET(GRAMMAR_STRING_CORE_DOUBLE "\"~([#DOUBLE_QUOTE##BACKSLASH#]) | (BACKSLASH .)\"") + SET(GRAMMAR_MULTILINE_COMMENT_CORE "\".\"") + + CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Java/Java.stg.in ${antlr_templates}/Java/Java.stg) + + IF(EXPORT_CMAKE) + MANUAL_GENERATION(${sql_grammar}) + ELSE() + CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in ${sql_grammar}) + ENDIF() + + RUN_ANTLR4( + ${sql_grammar} + -no-listener + -lib . + -o ${antlr_output} + -Dlanguage=Java + IN ${sql_grammar} ${antlr_templates}/Java/Java.stg + OUT_NOAUTO SQLv1Parser.proto + CWD ${antlr_output} + ) +ENDIF() + +SRCS(SQLv1Parser.proto) + +EXCLUDE_TAGS(GO_PROTO JAVA_PROTO) + +END() diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/epilogue.cmake b/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/epilogue.cmake new file mode 100644 index 000000000000..dc1496a45630 --- /dev/null +++ b/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/epilogue.cmake @@ -0,0 +1,9 @@ +set(GRAMMAR_STRING_CORE_SINGLE "~(['\\]) | (BACKSLASH .)") +set(GRAMMAR_STRING_CORE_DOUBLE "~([\"\\]) | (BACKSLASH .)") +set(GRAMMAR_MULTILINE_COMMENT_CORE ".") + +configure_file( + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in + ${CMAKE_BINARY_DIR}/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/SQLv1Antlr4.g +) + diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/update.sh b/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/update.sh new file mode 100755 index 000000000000..9e562ac57f2c --- /dev/null +++ b/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/update.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -eux +cp ../v1_proto/ya.make.gen ../v1_proto/ya.make +yag make ../v1_proto --add-result=".h" --add-result=".cc" +rm ../v1_proto/ya.make +python3 ../multiproto.py SQLv1Parser ../v1_proto . + diff --git a/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/ya.make b/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/ya.make new file mode 100644 index 000000000000..f23e8a916cb3 --- /dev/null +++ b/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4/ya.make @@ -0,0 +1,87 @@ +LIBRARY() + +SET(antlr_output ${ARCADIA_BUILD_ROOT}/${MODDIR}) +SET(antlr_templates ${antlr_output}/org/antlr/v4/tool/templates/codegen) +SET(sql_grammar ${antlr_output}/SQLv1Antlr4.g) + +SET(ANTLR_PACKAGE_NAME NSQLv1Generated) + +SET(GRAMMAR_STRING_CORE_SINGLE "\"~(['#BACKSLASH#]) | (BACKSLASH .)\"") +SET(GRAMMAR_STRING_CORE_DOUBLE "\"~([#DOUBLE_QUOTE##BACKSLASH#]) | (BACKSLASH .)\"") +SET(GRAMMAR_MULTILINE_COMMENT_CORE "\".\"") + +CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Java/Java.stg.in ${antlr_templates}/Java/Java.stg) + +IF(EXPORT_CMAKE) + MANUAL_GENERATION(${sql_grammar}) +ELSE() + CONFIGURE_FILE(${ARCADIA_ROOT}/ydb/library/yql/sql/v1/SQLv1Antlr4.g.in ${sql_grammar}) +ENDIF() + +RUN_ANTLR4( + ${sql_grammar} + -lib . + -no-listener + -o ${antlr_output} + -Dlanguage=Java + IN ${sql_grammar} ${antlr_templates}/Java/Java.stg + OUT_NOAUTO SQLv1Antlr4Parser.proto + CWD ${antlr_output} +) + +IF (USE_VANILLA_PROTOC) + SET(PROTOC_PATH contrib/tools/protoc_std) +ELSE() + SET(PROTOC_PATH contrib/tools/protoc/bin) +ENDIF() + + +RUN_PROGRAM( + $PROTOC_PATH -I=$CURDIR -I=$ARCADIA_ROOT -I=$ARCADIA_BUILD_ROOT -I=$ARCADIA_ROOT/contrib/libs/protobuf/src + --cpp_out=$ARCADIA_BUILD_ROOT --cpp_styleguide_out=$ARCADIA_BUILD_ROOT + --plugin=protoc-gen-cpp_styleguide=contrib/tools/protoc/plugins/cpp_styleguide + SQLv1Antlr4Parser.proto + IN SQLv1Antlr4Parser.proto + TOOL contrib/tools/protoc/plugins/cpp_styleguide + OUT_NOAUTO SQLv1Antlr4Parser.pb.h SQLv1Antlr4Parser.pb.cc + CWD $ARCADIA_BUILD_ROOT +) + +RUN_PYTHON3( + ${ARCADIA_ROOT}/ydb/library/yql/parser/proto_ast/gen/multiproto.py SQLv1Antlr4Parser + IN SQLv1Antlr4Parser.pb.h + IN SQLv1Antlr4Parser.pb.cc + OUT_NOAUTO + SQLv1Antlr4Parser.pb.code0.cc + SQLv1Antlr4Parser.pb.code1.cc + SQLv1Antlr4Parser.pb.code2.cc + SQLv1Antlr4Parser.pb.code3.cc + SQLv1Antlr4Parser.pb.code4.cc + SQLv1Antlr4Parser.pb.code5.cc + SQLv1Antlr4Parser.pb.code6.cc + SQLv1Antlr4Parser.pb.code7.cc + SQLv1Antlr4Parser.pb.code8.cc + SQLv1Antlr4Parser.pb.code9.cc + SQLv1Antlr4Parser.pb.data.cc + SQLv1Antlr4Parser.pb.classes.h + SQLv1Antlr4Parser.pb.main.h + CWD $ARCADIA_BUILD_ROOT/ydb/library/yql/parser/proto_ast/gen/v1_proto_split_antlr4 +) + +PEERDIR(contrib/libs/protobuf) + +SRCS( + SQLv1Antlr4Parser.pb.code0.cc + SQLv1Antlr4Parser.pb.code1.cc + SQLv1Antlr4Parser.pb.code2.cc + SQLv1Antlr4Parser.pb.code3.cc + SQLv1Antlr4Parser.pb.code4.cc + SQLv1Antlr4Parser.pb.code5.cc + SQLv1Antlr4Parser.pb.code6.cc + SQLv1Antlr4Parser.pb.code7.cc + SQLv1Antlr4Parser.pb.code8.cc + SQLv1Antlr4Parser.pb.code9.cc + SQLv1Antlr4Parser.pb.data.cc +) + +END() diff --git a/ydb/library/yql/parser/proto_ast/gen/ya.make b/ydb/library/yql/parser/proto_ast/gen/ya.make index 81d39789f9c8..347d52576ccf 100644 --- a/ydb/library/yql/parser/proto_ast/gen/ya.make +++ b/ydb/library/yql/parser/proto_ast/gen/ya.make @@ -4,4 +4,6 @@ RECURSE( v0_proto_split v1 v1_proto_split + v1_antlr4 + v1_proto_split_antlr4 ) diff --git a/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg.in b/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg.in new file mode 100644 index 000000000000..359d6c013cdc --- /dev/null +++ b/ydb/library/yql/parser/proto_ast/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg.in @@ -0,0 +1,1425 @@ +/* + * [The "BSD license"] + * Copyright (c) 2015 Dan McLaughlin, Mike Lischke + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import "Files.stg" // All file specific stuff. + +cppTypeInitMap ::= [ + "int":"0", + "long":"0", + "float":"0.0f", + "double":"0.0", + "bool":"false", + "short":"0", + "char":"0", + default: "nullptr" // anything other than a primitive type is an object +] + +LexerHeader(lexer, atn, actionFuncs, sempredFuncs, superClass = {antlr4::Lexer}) ::= << + + +class : public { +public: + + enum { + = }; separator=", ", wrap, anchor> + }; + + + + enum { + = }; separator=", ", wrap, anchor> + }; + + + + enum { + = }; separator=", ", wrap, anchor> + }; + + + explicit (antlr4::CharStream *input); + + ~() override; + + + + std::string getGrammarFileName() const override; + + const std::vector\& getRuleNames() const override; + + const std::vector\& getChannelNames() const override; + + const std::vector\& getModeNames() const override; + + const antlr4::dfa::Vocabulary& getVocabulary() const override; + + antlr4::atn::SerializedATNView getSerializedATN() const override; + + const antlr4::atn::ATN& getATN() const override; + + + void action(antlr4::RuleContext *context, size_t ruleIndex, size_t actionIndex) override; + + + + bool sempred(antlr4::RuleContext *_localctx, size_t ruleIndex, size_t predicateIndex) override; + + + // By default the static state used to implement the lexer is lazily initialized during the first + // call to the constructor. You can call this function if you wish to initialize the static state + // ahead of time. + static void initialize(); + +private: + + + // Individual action functions triggered by action() above. + + + // Individual semantic predicate functions triggered by sempred() above. + + + +}; +>> + +Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass = {Lexer}) ::= << + +using namespace antlr4; + +namespace { + +struct StaticData final { + StaticData(std::vector\ ruleNames, + std::vector\ channelNames, + std::vector\ modeNames, + std::vector\ literalNames, + std::vector\ symbolicNames) + : ruleNames(std::move(ruleNames)), channelNames(std::move(channelNames)), + modeNames(std::move(modeNames)), literalNames(std::move(literalNames)), + symbolicNames(std::move(symbolicNames)), + vocabulary(this->literalNames, this->symbolicNames) {} + + StaticData(const StaticData&) = delete; + StaticData(StaticData&&) = delete; + StaticData& operator=(const StaticData&) = delete; + StaticData& operator=(StaticData&&) = delete; + + std::vector\ decisionToDFA; + antlr4::atn::PredictionContextCache sharedContextCache; + const std::vector\ ruleNames; + const std::vector\ channelNames; + const std::vector\ modeNames; + const std::vector\ literalNames; + const std::vector\ symbolicNames; + const antlr4::dfa::Vocabulary vocabulary; + antlr4::atn::SerializedATNView serializedATN; + std::unique_ptr\ atn; +}; + +::antlr4::internal::OnceFlag LexerOnceFlag; +StaticData *LexerStaticData = nullptr; + +void LexerInitialize() { + assert(LexerStaticData == nullptr); + auto staticData = std::make_unique\<StaticData>( + std::vector\{ + "}; separator = ", ", wrap, anchor> + }, + std::vector\{ + "DEFAULT_TOKEN_CHANNEL", "HIDDEN", "}; separator = ", ", wrap, anchor> + }, + std::vector\{ + "}; separator = ", ", wrap, anchor> + }, + std::vector\{ + }; null = "\"\"", separator = ", ", wrap, anchor> + }, + std::vector\{ + }; null = "\"\"", separator = ", ", wrap, anchor> + } + ); + + LexerStaticData = staticData.release(); +} + +} + +::(CharStream *input) : (input) { + ::initialize(); + _interpreter = new atn::LexerATNSimulator(this, *LexerStaticData->atn, LexerStaticData->decisionToDFA, LexerStaticData->sharedContextCache); +} + +::~() { + delete _interpreter; +} + +std::string ::getGrammarFileName() const { + return ""; +} + +const std::vector\& ::getRuleNames() const { + return LexerStaticData->ruleNames; +} + +const std::vector\& ::getChannelNames() const { + return LexerStaticData->channelNames; +} + +const std::vector\& ::getModeNames() const { + return LexerStaticData->modeNames; +} + +const dfa::Vocabulary& ::getVocabulary() const { + return LexerStaticData->vocabulary; +} + +antlr4::atn::SerializedATNView ::getSerializedATN() const { + return LexerStaticData->serializedATN; +} + +const atn::ATN& ::getATN() const { + return *LexerStaticData->atn; +} + + + + +void ::action(RuleContext *context, size_t ruleIndex, size_t actionIndex) { + switch (ruleIndex) { + : Action(antlrcpp::downCast\< *>(context), actionIndex); break;}; separator="\n"> + + default: + break; + } +} + + + +bool ::sempred(RuleContext *context, size_t ruleIndex, size_t predicateIndex) { + switch (ruleIndex) { + : return Sempred(antlrcpp::downCast\< *>(context), predicateIndex);}; separator="\n"> + + default: + break; + } + return true; +} + + + + + + +void ::initialize() { + ::antlr4::internal::call_once(LexerOnceFlag, LexerInitialize); +} +>> + +RuleActionFunctionHeader(r, actions) ::= << +void Action( *context, size_t actionIndex); +>> + +RuleActionFunction(r, actions) ::= << +void ::Action( *context, size_t actionIndex) { + switch (actionIndex) { + : break;}; separator="\n"> + + default: + break; + } +} + +>> + +RuleSempredFunctionHeader(r, actions) ::= << +bool Sempred( *_localctx, size_t predicateIndex); +>> + +RuleSempredFunction(r, actions) ::= << + +bool ::Sempred( *_localctx, size_t predicateIndex) { + switch (predicateIndex) { + : return }; separator=";\n">; + + default: + break; + } + return true; +} + +>> + +//-------------------------------------------------------------------------------------------------- + +ParserHeader(parser, funcs, atn, sempredFuncs, superClass = {antlr4::Parser}) ::= << + + +class : public { +public: + typedef google::protobuf::Message* RuleReturnType; + + + enum { + = }; separator=", ", wrap, anchor> + }; + + + + enum { + = }; separator=", ", wrap, anchor> + }; + + + explicit (antlr4::TokenStream *input); + + (antlr4::TokenStream *input, const antlr4::atn::ParserATNSimulatorOptions &options); + (antlr4::TokenStream *input, google::protobuf::Arena* arena); + RuleReturnType Parse(NProtoAST::IErrorCollector* errors = 0); + + ~() override; + + std::string getGrammarFileName() const override; + + const antlr4::atn::ATN& getATN() const override; + + const std::vector\& getRuleNames() const override; + + const antlr4::dfa::Vocabulary& getVocabulary() const override; + + antlr4::atn::SerializedATNView getSerializedATN() const override; + + + + Context;}; separator = "\n"> + + + + + bool sempred(antlr4::RuleContext *_localctx, size_t ruleIndex, size_t predicateIndex) override; + + + + + // By default the static state used to implement the parser is lazily initialized during the first + // call to the constructor. You can call this function if you wish to initialize the static state + // ahead of time. + static void initialize(); + bool error = false; + static const size_t TOKEN_EOF = std::numeric_limits\::max(); + +private: + google::protobuf::Arena* Arena = nullptr; + +}; +>> + +Parser(parser, funcs, atn, sempredFuncs, superClass = {Parser}) ::= << + +using namespace antlr4; + +namespace { + +struct StaticData final { + StaticData(std::vector\ ruleNames, + std::vector\ literalNames, + std::vector\ symbolicNames) + : ruleNames(std::move(ruleNames)), literalNames(std::move(literalNames)), + symbolicNames(std::move(symbolicNames)), + vocabulary(this->literalNames, this->symbolicNames) {} + + StaticData(const StaticData&) = delete; + StaticData(StaticData&&) = delete; + StaticData& operator=(const StaticData&) = delete; + StaticData& operator=(StaticData&&) = delete; + + std::vector\ decisionToDFA; + antlr4::atn::PredictionContextCache sharedContextCache; + const std::vector\ ruleNames; + const std::vector\ literalNames; + const std::vector\ symbolicNames; + const antlr4::dfa::Vocabulary vocabulary; + antlr4::atn::SerializedATNView serializedATN; + std::unique_ptr\ atn; +}; + +::antlr4::internal::OnceFlag ParserOnceFlag; +StaticData *ParserStaticData = nullptr; + +void ParserInitialize() { + assert(ParserStaticData == nullptr); + auto staticData = std::make_unique\<StaticData>( + std::vector\{ + "}; separator = ", ", wrap, anchor> + }, + std::vector\{ + }; null = "\"\"", separator = ", ", wrap, anchor> + }, + std::vector\{ + }; null = "\"\"", separator = ", ", wrap, anchor> + } + ); + + ParserStaticData = staticData.release(); +} + +} + +::(TokenStream *input) : (input, antlr4::atn::ParserATNSimulatorOptions()) {} + +::(antlr4::TokenStream *input, google::protobuf::Arena* arena) : (input, antlr4::atn::ParserATNSimulatorOptions()) { + Arena = arena; +} + +::(TokenStream *input, const antlr4::atn::ParserATNSimulatorOptions &options) : (input) { + ::initialize(); + _interpreter = new atn::ParserATNSimulator(this, *ParserStaticData->atn, ParserStaticData->decisionToDFA, ParserStaticData->sharedContextCache, options); +} + +::RuleReturnType ::Parse(NProtoAST::IErrorCollector* errors) { + try { + RuleReturnType root(::()); + if (!error && root) { + auto ast = google::protobuf::Arena::CreateMessage\<@ANTLR_PACKAGE_NAME@::TSQLv1ParserAST>(Arena); + Y_ASSERT(dynamic_cast\<@ANTLR_PACKAGE_NAME@::TRule_* >(root)); + ast->unsafe_arena_set_allocated_rule_(static_cast\<@ANTLR_PACKAGE_NAME@::TRule_* >(root)); + return ast; + \} + return nullptr; + \} catch (RecognitionException &err) { + return nullptr; + \} + return nullptr; +} + +::~() { + delete _interpreter; +} + +const atn::ATN& ::getATN() const { + return *ParserStaticData->atn; +} + +std::string ::getGrammarFileName() const { + return ""; +} + +const std::vector\& ::getRuleNames() const { + return ParserStaticData->ruleNames; +} + +const dfa::Vocabulary& ::getVocabulary() const { + return ParserStaticData->vocabulary; +} + +antlr4::atn::SerializedATNView ::getSerializedATN() const { + return ParserStaticData->serializedATN; +} + + + + + + +bool ::sempred(RuleContext *context, size_t ruleIndex, size_t predicateIndex) { + switch (ruleIndex) { + : return Sempred(antlrcpp::downCast\< *>(context), predicateIndex);}; separator="\n"> + + default: + break; + } + return true; +} + + + +void ::initialize() { + ::antlr4::internal::call_once(ParserOnceFlag, ParserInitialize); +} +>> + +SerializedATNHeader(model) ::= << +>> + +SerializedATN(model) ::= << +static const int32_t serializedATNSegment[] = { + }; separator=",", wrap> +}; +staticData->serializedATN = antlr4::atn::SerializedATNView(serializedATNSegment, sizeof(serializedATNSegment) / sizeof(serializedATNSegment[0])); + +antlr4::atn::ATNDeserializer deserializer; +staticData->atn = deserializer.deserialize(staticData->serializedATN); + +const size_t count = staticData->atn->getNumberOfDecisions(); +staticData->decisionToDFA.reserve(count); +for (size_t i = 0; i \< count; i++) { + staticData->decisionToDFA.emplace_back(staticData->atn->getDecisionState(i), i); +} +>> + +RuleFunctionHeader(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedActions, finallyAction, postamble, exceptions) ::= << + +}; separator="\n"> +RuleReturnType (); + +>> + +RuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedActions, finallyAction, postamble, exceptions) ::= << + +}; separator = "\n"> +::RuleReturnType ::() { + *_localctx = _tracker.createInstance\<\>(_ctx, getState()}>); + // proto stuff + @ANTLR_PACKAGE_NAME@::TRule_* retval = google::protobuf::Arena::CreateMessage\<@ANTLR_PACKAGE_NAME@::TRule_\>(Arena); + auto parent = retval; + std::string invoked_stmt; + // proto end + enterRule(_localctx, , ::Rule); + + + +#if __cplusplus > 201703L + auto onExit = finally([=, this] { +#else + auto onExit = finally([=] { +#endif + + exitRule(); + }); + try { + + size_t alt; + + + + + }; separator="\n"> + if (parent) { + parent->SetDescr(""); + } + + + + } + + + + catch (RecognitionException &e) { + _errHandler->reportError(this, e); + // _localctx->exception = std::current_exception(); + //_errHandler->recover(this, _localctx->exception); + throw; + } + + + if (retval && (!parent || !retval->Alt_case())) { + retval = nullptr; + } + + if (!parent) { + retval = nullptr; + } + + + return retval; +} +>> + +LeftRecursiveRuleFunctionHeader(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedActions, finallyAction, postamble) ::= << + +}; separator="\n"> +* (); +* (int precedence}>); +>> + +LeftRecursiveRuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs, namedActions, finallyAction, postamble) ::= << + +}; separator="\n"> + +::* ::() { + return (0}>); +} + +::* ::(int precedence}>) { + ParserRuleContext *parentContext = _ctx; + size_t parentState = getState(); + // proto stuff + @ANTLR_PACKAGE_NAME@::TRule_* retval = google::protobuf::Arena::CreateMessage\<@ANTLR_PACKAGE_NAME@::TRule_\>(Arena); + auto parent = retval; + std::string invoked_stmt; + // proto end + :: *_localctx = _tracker.createInstance\<\>(_ctx, parentState}>); + :: *previousContext = _localctx; + (void)previousContext; // Silence compiler, in case the context is not used by generated code. + size_t startState = ; + enterRecursionRule(_localctx, , ::Rule, precedence); + + + + +#if __cplusplus > 201703L + auto onExit = finally([=, this] { +#else + auto onExit = finally([=] { +#endif + + unrollRecursionContexts(parentContext); + }); + try { + size_t alt; + + }; separator="\n"> + + + + + + }; separator="\n"> + + + + + + + + } + catch (RecognitionException &e) { + // _errHandler->reportError(this, e); + // _localctx->exception = std::current_exception(); + //_errHandler->recover(this, _localctx->exception); + throw; + } + + if (!parent) { + retval = nullptr; + } + + + + if (!parent || !retval->Alt_case()) { + retval = nullptr; + } + + if (!parent) { + retval = nullptr; + } + + + if (!parent) { + retval = nullptr; + } + + + return retval; +} +>> + +StructDeclHeader(struct, ctorAttrs, attrs, getters, dispatchMethods, interfaces, extensionMembers) ::= << +class : public antlr4::ParserRuleContext, { +public: + ;}; separator = "\n"> + (antlr4::ParserRuleContext *parent, size_t invokingState); + (antlr4::ParserRuleContext *parent, size_t invokingState}>); + + () = default; + void copyFrom( *context); + using antlr4::ParserRuleContext::copyFrom; + + + virtual size_t getRuleIndex() const override; + }; separator = "\n"> + + + +}; + +>> + +element(e, elemId) ::= << + +>> + +StructDecl(struct, ctorAttrs, attrs, getters, dispatchMethods, interfaces, extensionMembers) ::= << +//----------------- ------------------------------------------------------------------ + + +::::(ParserRuleContext *parent, size_t invokingState) + : ParserRuleContext(parent, invokingState) { +} + + +::::(ParserRuleContext *parent, size_t invokingState}>) + : ParserRuleContext(parent, invokingState) { + = ;}; separator="\n"> +} + +}; separator = "\n"> + +size_t ::::getRuleIndex() const { + return ::Rule; +} + + +void ::::copyFrom( *ctx) { + ParserRuleContext::copyFrom(ctx); + = ctx->;}; separator = "\n"> +} + + + + +>> + +AltLabelStructDeclHeader(struct, attrs, getters, dispatchMethods) ::= << +class : public Context { +public: + (Context *ctx); + + ;}; separator = "\n"> + }; separator = "\n"> + +}; + +>> + +AltLabelStructDecl(struct, attrs, getters, dispatchMethods) ::= << +//----------------- ------------------------------------------------------------------ + +}; separator = "\n"> +}; separator = "\n"> +::::(Context *ctx) { copyFrom(ctx); } + + +>> + + +CodeBlockForOuterMostAltHeader(currentOuterMostAltCodeBlock, locals, preamble, ops) ::= "" +CodeBlockForOuterMostAlt(currentOuterMostAltCodeBlock, locals, preamble, ops) ::= << + +_localctx = _tracker.createInstance\<::Context>(_localctx); + +enterOuterAlt(_localctx, ); + +>> + +CodeBlockForAltHeader(currentAltCodeBlock, locals, preamble, ops) ::= "" +CodeBlockForAlt(currentAltCodeBlock, locals, preamble, ops) ::= << + + +}; separator="\n"> +>> + +LL1AltBlockHeader(choice, preamble, alts, error) ::= "" +LL1AltBlock(choice, preamble, alts, error) ::= << +{ // LL1AltBlock + setState(); + _errHandler->sync(this); + LL1AltBlock(choice, preamble, alts, error) = _input->LT(1); + + switch (_input->LA(1)) { + { + + // LL1AltBlock protobuf stuff + auto &global_parent = parent; + auto parent = global_parent->MutableBlock()->MutableAlt_(); + + global_parent->MutableBlock()->SetDescr(""); + + // protobuf end + + + + global_parent->MutableAlt_()->SetDescr(invoked_stmt.c_str()); + + + // LL1AltBlock protobuf stuff + if (!parent) { + global_parent->ClearBlock(); + global_parent = nullptr; + \} + // protobuf end + + break; + \} + }; separator = "\n"> + default: + + } +} +>> + +LL1OptionalBlockHeader(choice, alts, error) ::= "" +LL1OptionalBlock(choice, alts, error) ::= << +{ // LL1OptionalBlock + setState(); + _errHandler->sync(this); + switch (_input->LA(1)) { + { + + // LL1OptionalBlock protobuf stuff + auto &global_parent = parent; + auto parent = global_parent->MutableBlock()->MutableAlt(); + + global_parent->MutableBlock()->SetDescr(""); + + // protobuf end + + + + // LL1OptionalBlock protobuf stuff + if (!parent) { + global_parent->ClearBlock(); + global_parent = nullptr; + \} + // protobuf end + + break; + \} + }; separator="\n"> + default: + break; + \} +} +>> + +LL1OptionalBlockSingleAltHeader(choice, expr, alts, preamble, error, followExpr) ::= "" +LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= << +setState(); +_errHandler->sync(this); + + +if () { + // LL1OptionalBlockSingleAlt protobuf stuff + auto &global_parent = parent; + auto parent = global_parent->MutableBlock(); + global_parent->MutableBlock()->SetDescr(""); + // protobuf end + + // LL1OptionalBlockSingleAlt protobuf stuff + if (!parent) { + global_parent->ClearBlock(); + global_parent = nullptr; + \} + // protobuf end +} +>> + +LL1StarBlockSingleAltHeader(choice, loopExpr, alts, preamble, iteration) ::= "" +LL1StarBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= << +{ // LL1StarBlockSingleAlt + setState(); + _errHandler->sync(this); + + // LL1StarBlockSingleAlt protobuf stuff + auto global_parent = parent; + // protobuf end + while () { + // LL1StarBlockSingleAlt protobuf stuff + auto parent = global_parent->AddBlock(); + // protobuf end + + // LL1StarBlockSingleAlt protobuf stuff + if (!parent) { + global_parent->MutableBlock()->RemoveLast(); + \} + // protobuf end + setState(); + _errHandler->sync(this); + + } +} +>> + +LL1PlusBlockSingleAltHeader(choice, loopExpr, alts, preamble, iteration) ::= "" +LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= << +{ + setState(); + _errHandler->sync(this); + + // LL1PlusBlockSingleAlt protobuf stuff + auto global_parent = parent; + // protobuf end + do { + // LL1StarBlockSingleAlt protobuf stuff + auto parent = global_parent->AddBlock(); + // protobuf end + + // LL1StarBlockSingleAlt protobuf stuff + if (!parent) { + global_parent->MutableBlock()->RemoveLast(); + \} + // protobuf end + setState(); + _errHandler->sync(this); + + } while (); + Y_ABORT_UNLESS(global_parent, "parentBlock is null"); + if (!global_parent->BlockSize()) { + parent = nullptr; + \} +} +>> + +// LL(*) stuff + +AltBlockHeader(choice, preamble, alts, error) ::= "" +AltBlock(choice, preamble, alts, error) ::= << +{ // AltBlock + setState(); + _errHandler->sync(this); + = _input->LT(1); + + switch (getInterpreter\()->adaptivePredict(_input, , _ctx)) { + : { + + // AltBlock protobuf stuff + auto &global_parent = parent; + auto parent = global_parent->MutableBlock()->MutableAlt_(); + + global_parent->MutableBlock()->SetDescr(""); + + // protobuf end + + + + global_parent->MutableAlt_()->SetDescr(invoked_stmt.c_str()); + + + // AltBlock protobuf stuff + if (!parent) { + global_parent->ClearBlock(); + global_parent = nullptr; + \} + // protobuf end + + break; + \} + }; separator="\n"> + default: + break; + } +} +>> + +OptionalBlockHeader(choice, alts, error) ::= "" +OptionalBlock(choice, alts, error) ::= << +{ // OptionalBlock + setState(); + _errHandler->sync(this); + switch (getInterpreter\()->adaptivePredict(_input, , _ctx)) { + + 1: { + + // OptionalBlock protobuf stuff + auto &global_parent = parent; + auto parent = global_parent->MutableBlock()->MutableAlt(); + + global_parent->MutableBlock()->SetDescr(""); + + // protobuf end + + + + // LL1OptionalBlock protobuf stuff + if (!parent) { + global_parent->ClearBlock(); + global_parent = nullptr; + \} + // protobuf end + + break; + \} + }; separator = "\n"> + default: + break; + } +} +>> + +StarBlockHeader(choice, alts, sync, iteration) ::= "" +StarBlock(choice, alts, sync, iteration) ::= << +{ // StarBlock + setState(); + _errHandler->sync(this); + alt = getInterpreter\()->adaptivePredict(_input, , _ctx); + // StarBlock protobuf stuff + auto& global_parent = parent; + // protobuf end + while (alt != && alt != atn::ATN::INVALID_ALT_NUMBER) { + if (alt == 1 + 1) { + + // StarBlock protobuf stuff + auto parent = global_parent->AddBlock(); + // protobuf end + + // StarBlock protobuf stuff + if (!parent) { + global_parent->MutableBlock()->RemoveLast(); + } + // protobuf end + } + setState(); + _errHandler->sync(this); + alt = getInterpreter\()->adaptivePredict(_input, , _ctx); + } +} +>> + +PlusBlockHeader(choice, alts, error) ::= "" +PlusBlock(choice, alts, error) ::= << +{ // PlusBlock + setState(); + _errHandler->sync(this); + alt = 1 + 1; + // PlusBlock protobuf stuff + auto global_parent = parent; + // protobuf end + do { + switch (alt) { + + 1: { + + // OptionalBlock protobuf stuff + auto parent = global_parent->AddBlock()->MutableAlt(); + // protobuf end + + auto parent = global_parent->AddBlock(); + + + if (!parent) { + global_parent->MutableBlock()->RemoveLast(); + \} + break; + \} + }; separator="\n"> + default: + + } + setState(); + _errHandler->sync(this); + alt = getInterpreter\()->adaptivePredict(_input, , _ctx); + } while (alt != && alt != atn::ATN::INVALID_ALT_NUMBER); + Y_ABORT_UNLESS(global_parent, "parentBlock is null"); + if (!global_parent->BlockSize()) { + parent = nullptr; + \} +} +>> + +Sync(s) ::= "Sync(s) sync();" + +ThrowNoViableAltHeader(t) ::= "" +ThrowNoViableAlt(t) ::= "throw NoViableAltException(this);" + +TestSetInlineHeader(s) ::= "" +TestSetInline(s) ::= << +}; separator=" || "> +>> + +// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test +testShiftInRange(shiftAmount) ::= << +(( & ~ 0x3fULL) == 0) +>> + +bitsetBitfieldComparison(s, bits) ::= << +})> && + ((1ULL \<\< ) & ) != 0 +>> + +isZero ::= [ + "0": true, + default: false +] + +offsetShift(shiftAmount, offset, prefix = false) ::= <% +(::TOKEN_ - )::TOKEN_ +%> + +bitsetInlineComparison(s, bits) ::= <% + == ::TOKEN_}; separator = "\n\n|| "> +%> + +cases(tokens) ::= << +::TOKEN_:}; separator="\n"> +>> + +InvokeRuleHeader(r, argExprsChunks) ::= "InvokeRuleHeader" +InvokeRule(r, argExprsChunks) ::= << +{ + setState(); + invoked_stmt = ""; + auto res = = }>(,); + if (parent && res) { + Y_ASSERT(dynamic_cast\<@ANTLR_PACKAGE_NAME@::TRule_* >(res)); + Y_ABORT_UNLESS(parent, "Parent is null"); + parent->unsafe_arena_set_allocated_rule_(static_cast\<@ANTLR_PACKAGE_NAME@::TRule_* >(res)); + \} else { + parent = nullptr; + } +\} +>> + +MatchTokenHeader(m) ::= "" +MatchToken(m) ::= << +{ + setState(); + auto token = = }>match(::TOKEN_); + if (parent && token && token->getType() > 0) { + Y_ABORT_UNLESS(parent, "Parent is null"); + parent->MutableToken()->SetId(token->getType()); + parent->MutableToken()->SetLine(token->getLine()); + parent->MutableToken()->SetColumn(token->getCharPositionInLine()); + parent->MutableToken()->SetElement(); + parent->MutableToken()->SetValue(token->getText().c_str()); + \} else { + parent = nullptr; + } +} +>> + +MatchSetHeader(m, expr, capture) ::= "" +MatchSet(m, expr, capture) ::= "" + +MatchNotSetHeader(m, expr, capture) ::= "" +MatchNotSet(m, expr, capture) ::= "" + +CommonSetStuff(m, expr, capture, invert) ::= << +setState(); + = }>_input->LT(1); + +if ( == 0 || == Token::EOF || !()) { + = }>_errHandler->recoverInline(this); +} +else { + auto token = _input->LT(1); + if (token && token->getType() > 0) { + Y_ABORT_UNLESS(parent, "Parent is null"); + parent->MutableToken()->SetId(token->getType()); + parent->MutableToken()->SetLine(token->getLine()); + parent->MutableToken()->SetColumn(token->getCharPositionInLine()); + parent->MutableToken()->SetElement(); + parent->MutableToken()->SetValue(token->getText().c_str()); + } + _errHandler->reportMatch(this); + consume(); +} +>> + +WildcardHeader(w) ::= "" +Wildcard(w) ::= << +setState(); + = }>matchWildcard(); +>> + +// ACTION STUFF + +ActionHeader(a, foo, chunks) ::= "" +Action(a, foo, chunks) ::= "" + +ArgAction(a, chunks) ::= "ArgAction(a, chunks) " + +SemPredHeader(p, chunks, failChunks) ::= "" +SemPred(p, chunks, failChunks) ::= << +setState(); + +if (!()) throw FailedPredicateException(this, , , ); +>> + +ExceptionClauseHeader(e, catchArg, catchAction) ::= "" +ExceptionClause(e, catchArg, catchAction) ::= << +catch () { + +} +>> + +// Lexer actions are not associated with model objects. + +LexerSkipCommand() ::= "skip();" +LexerMoreCommand() ::= "more();" +LexerPopModeCommand() ::= "popMode();" + +LexerTypeCommand(arg, grammar) ::= "type = ::;" +LexerChannelCommand(arg, grammar) ::= "channel = ;" +LexerModeCommand(arg, grammar) ::= "mode = Mode::;" +LexerPushModeCommand(arg, grammar) ::= "pushMode(Mode::);" + +ActionTextHeader(t) ::= "" +ActionText(t) ::= "" + +ActionTemplateHeader(t) ::= "" +ActionTemplate(t) ::= "" + +ArgRefHeader(t) ::= "" +ArgRef(a) ::= "_localctx->" + +LocalRefHeader(t) ::= "" +LocalRef(a) ::= "_localctx->" + +RetValueRefHeader(t) ::= "" +RetValueRef(a) ::= "_localctx->" + +QRetValueRefHeader(t) ::= "" +QRetValueRef(a) ::= "->->" +/** How to translate $tokenLabel */ + +TokenRefHeader(t) ::= "" +TokenRef(t) ::= "->" + +LabelRefHeader(t) ::= "" +LabelRef(t) ::= "->" + +ListLabelRefHeader(t) ::= "" +ListLabelRef(t) ::= "->" + +SetAttrHeader(t) ::= "" +SetAttr(s,rhsChunks) ::= "-> = ;" + +InputSymbolType() ::= " *" + +TokenPropertyRef_textHeader(t) ::= "" +TokenPropertyRef_text(t) ::= <<(-> != nullptr ? ->->getText() : "")>> + +TokenPropertyRef_typeHeader(t) ::= "" +TokenPropertyRef_type(t) ::= "(-> != nullptr ? ->->getType() : 0)" + +TokenPropertyRef_lineHeader(t) ::= "" +TokenPropertyRef_line(t) ::= "(-> != nullptr ? ->->getLine() : 0)" + +TokenPropertyRef_posHeader(t) ::= "" +TokenPropertyRef_pos(t) ::= "(-> != nullptr ? ->->getCharPositionInLine() : 0)" + +TokenPropertyRef_channelHeader(t) ::= "" +TokenPropertyRef_channel(t) ::= "(-> != nullptr ? ->->getChannel() : 0)" + +TokenPropertyRef_indexHeader(t) ::= "" +TokenPropertyRef_index(t) ::= "(-> != nullptr ? ->->getTokenIndex() : 0)" + +TokenPropertyRef_intHeader(t) ::= "" +TokenPropertyRef_int(t) ::= "(-> != nullptr ? std::stoi(->->getText()) : 0)" + +RulePropertyRef_startHeader(r) ::= "" +RulePropertyRef_start(r) ::= "(-> != nullptr ? (->->start) : nullptr)" + +RulePropertyRef_stopHeader(r) ::= "" +RulePropertyRef_stop(r) ::= "(-> != nullptr ? (->->stop) : nullptr)" + +RulePropertyRef_textHeader(r) ::= "" +RulePropertyRef_text(r) ::= "(-> != nullptr ? _input->getText(->->start, ->->stop) : nullptr)" + +RulePropertyRef_ctxHeader(r) ::= "" +RulePropertyRef_ctx(r) ::= "->" + +ThisRulePropertyRef_start(r) ::= "ThisRulePropertyRef_start(r) _localctx->start" +ThisRulePropertyRef_stop(r) ::= "ThisRulePropertyRef_stop(r) _localctx->stop" + +ThisRulePropertyRef_textHeader(r) ::= "" +ThisRulePropertyRef_text(r) ::= "_input->getText(_localctx->start, _input->LT(-1))" + +ThisRulePropertyRef_ctxHeader(r) ::= "" +ThisRulePropertyRef_ctx(r) ::= "_localctx" + +ThisRulePropertyRef_parserHeader(r) ::= "" +ThisRulePropertyRef_parser(r) ::= "this" + +NonLocalAttrRef(s) ::= "NonLocalAttrRef(s) ((Context)getInvokingContext())." +SetNonLocalAttr(s, rhsChunks) ::= + "SetNonLocalAttr(s, rhsChunks) ((Context)getInvokingContext()). = ;" + +AddToLabelListHeader(a) ::= "" +AddToLabelList(a) ::= << +->.push_back(); +>> + +TokenLabelType() ::= " *" + +TokenDeclHeader(t) ::= "antlr4:: = nullptr" +TokenDecl(t) ::= "" + +TokenTypeDeclHeader(t) ::= "" +TokenTypeDecl(t) ::= "size_t = 0;" + +TokenListDeclHeader(t) ::= "std::vector\ " +TokenListDecl(t) ::= "" + +RuleContextDeclHeader(r) ::= ":: * = nullptr" +RuleContextDecl(r) ::= "" + +RuleContextListDeclHeader(rdecl) ::= "std::vector\< *> " +RuleContextListDecl(rdecl) ::= "" + +ContextTokenGetterDeclHeader(t) ::= "antlr4::tree::TerminalNode *TOKEN_();" +ContextTokenGetterDecl(t) ::= << +tree::TerminalNode* ::::TOKEN_() { + return getToken(::TOKEN_, 0); +} + +>> + +ContextTokenListGetterDeclHeader(t) ::= "std::vector\ TOKEN_();" +ContextTokenListGetterDecl(t) ::= << +std::vector\ ::::TOKEN_() { + return getTokens(::TOKEN_); +} + +>> + +ContextTokenListIndexedGetterDeclHeader(t) ::= "antlr4::tree::TerminalNode* TOKEN_(size_t i);" +ContextTokenListIndexedGetterDecl(t) ::= << +tree::TerminalNode* ::::TOKEN_(size_t i) { + return getToken(::TOKEN_, i); +} + +>> + +ContextRuleGetterDeclHeader(r) ::= " *();" +ContextRuleGetterDecl(r) ::= << + +::* ::::() { + return getRuleContext\<::\>(0); +} + +>> + +ContextRuleListGetterDeclHeader(r) ::= "std::vector\< *> ();" +ContextRuleListGetterDecl(r) ::= << +std::vector\<:: *> ::::() { + return getRuleContexts\<::\>(); +} + +>> + +ContextRuleListIndexedGetterDeclHeader(r) ::= "* (size_t i);" +ContextRuleListIndexedGetterDecl(r) ::= << +::* ::::(size_t i) { + return getRuleContext\<::\>(i); +} + +>> + +LexerRuleContext() ::= "antlr4::RuleContext" + +// The rule context name is the rule followed by a suffix; e.g. r becomes rContext. +RuleContextNameSuffix() ::= "Context" + +ImplicitTokenLabel(tokenName) ::= << +Token +>> + +ImplicitRuleLabel(ruleName) ::= "Context" +ImplicitSetLabel(id) ::= "_tset" +ListLabelName(label) ::= "