From a03219348b38ede51a9d833c5d6e1f2b3d0c7d38 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Tue, 24 Jan 2023 21:54:59 -0600 Subject: [PATCH 1/4] In parser_binary_test, diff serialized protos --- test/parser_binary_test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/parser_binary_test.sh b/test/parser_binary_test.sh index d3885ef..863555e 100755 --- a/test/parser_binary_test.sh +++ b/test/parser_binary_test.sh @@ -5,12 +5,14 @@ echo "Local protos:" LOCAL_PROTOS=$(find ./test/resources -name "*.proto" | sort) for f in $LOCAL_PROTOS; do echo $f - ./src/parser_binary $f > /dev/null + ./src/parser_binary $f > "${f}_serialized.proto" + diff --side-by-side --ignore-all-space $f "${f}_serialized.proto" done echo "Google protos:" GOOGLE_PROTOS=$(find ./external/com_google_protobuf/src/google/protobuf -name "*.proto" | xargs grep --files-without-match "proto2" | sort) for f in $GOOGLE_PROTOS; do echo $f - ./src/parser_binary $f > /dev/null + ./src/parser_binary $f > "${f}_serialized.proto" + diff --side-by-side --ignore-all-space $f "${f}_serialized.proto" done From 6aa5521b9a7cf577c28ace8dc210e6f3285c6f84 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Tue, 24 Jan 2023 22:01:57 -0600 Subject: [PATCH 2/4] Emit syntax in deterministic place --- src/parser.py | 1 + src/proto_file.py | 2 +- src/proto_syntax.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/parser.py b/src/parser.py index 043097a..8a815f8 100644 --- a/src/parser.py +++ b/src/parser.py @@ -70,6 +70,7 @@ def parse_syntax_and_preceding_comments( raise ParseError(f"Proto doesn't have parseable syntax:\n{proto_content}") syntax = match_result.node proto_content = match_result.remaining_source.strip() + parsed_tree.append(syntax) return syntax, parsed_tree, proto_content diff --git a/src/proto_file.py b/src/proto_file.py index d02291b..3b51fa5 100644 --- a/src/proto_file.py +++ b/src/proto_file.py @@ -26,7 +26,7 @@ def options(self) -> list[ProtoOption]: return [node for node in self.nodes if isinstance(node, ProtoOption)] def serialize(self) -> str: - serialized_parts = [self.syntax.serialize()] + serialized_parts = [] previous_type = self.syntax.__class__ for node in self.nodes: # Attempt to group up lines of the same type. diff --git a/src/proto_syntax.py b/src/proto_syntax.py index 7e00991..c1f56e8 100644 --- a/src/proto_syntax.py +++ b/src/proto_syntax.py @@ -15,7 +15,7 @@ def __init__(self, syntax: ProtoStringLiteral): self.syntax = syntax def __eq__(self, other) -> bool: - return self.syntax == other.syntax + return isinstance(other, ProtoSyntax) and self.syntax == other.syntax def __str__(self) -> str: return f"" From 77e79f58090b7a457236dcc53794c724b53c5975 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Tue, 24 Jan 2023 22:02:08 -0600 Subject: [PATCH 3/4] Factor out proto comparison --- test/parser_binary_test.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/parser_binary_test.sh b/test/parser_binary_test.sh index 863555e..e8bc822 100755 --- a/test/parser_binary_test.sh +++ b/test/parser_binary_test.sh @@ -1,18 +1,21 @@ #!/usr/bin/env bash set -euo pipefail +function compare_proto() { + # Takes one argument, the relative path to the proto to compare. + echo $1 + ./src/parser_binary $1 > "${1}_serialized.proto" + diff --side-by-side --ignore-all-space --ignore-blank-lines $1 "${1}_serialized.proto" +} + echo "Local protos:" LOCAL_PROTOS=$(find ./test/resources -name "*.proto" | sort) for f in $LOCAL_PROTOS; do - echo $f - ./src/parser_binary $f > "${f}_serialized.proto" - diff --side-by-side --ignore-all-space $f "${f}_serialized.proto" + compare_proto $f done echo "Google protos:" GOOGLE_PROTOS=$(find ./external/com_google_protobuf/src/google/protobuf -name "*.proto" | xargs grep --files-without-match "proto2" | sort) for f in $GOOGLE_PROTOS; do - echo $f - ./src/parser_binary $f > "${f}_serialized.proto" - diff --side-by-side --ignore-all-space $f "${f}_serialized.proto" + compare_proto $f done From 2009b32cbd325fdc08843e431ba5868708d195e1 Mon Sep 17 00:00:00 2001 From: Charles Guo Date: Tue, 13 Feb 2024 23:25:20 -0500 Subject: [PATCH 4/4] Restore changes --- src/proto_file.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/proto_file.py b/src/proto_file.py index 94d793a..b8eabda 100644 --- a/src/proto_file.py +++ b/src/proto_file.py @@ -136,9 +136,9 @@ def construct( @staticmethod def parse_syntax_and_preceding_comments( proto_content: str, - ) -> tuple[ProtoSyntax, Sequence[ProtoComment], str]: + ) -> tuple[ProtoSyntax, Sequence[ProtoNode], str]: # First, parse any preceding comments. - parsed_tree = [] + parsed_tree: list[ProtoNode] = [] while True: for node_type in [ProtoSingleLineComment, ProtoMultiLineComment]: try: @@ -161,6 +161,7 @@ def parse_syntax_and_preceding_comments( raise ValueError(f"Proto doesn't have parseable syntax:\n{proto_content}") syntax = syntax_match.node proto_content = syntax_match.remaining_source.strip() + parsed_tree.append(syntax) return syntax, parsed_tree, proto_content @@ -172,7 +173,7 @@ def normalize(self) -> Optional["ProtoNode"]: ) def serialize(self) -> str: - serialized_parts = [self.syntax.serialize()] + serialized_parts = [] previous_type: type[ProtoNode] = self.syntax.__class__ for node in self.nodes: # Attempt to group up lines of the same type.