From 86ee074678eeab786dc80cbc2c1c7d0efd763a98 Mon Sep 17 00:00:00 2001 From: Boshen Date: Tue, 12 Mar 2024 17:51:22 +0800 Subject: [PATCH] fix(parser): remove all duplicated comments in trivia builder (#2689) --- crates/oxc_parser/src/lexer/mod.rs | 6 +-- crates/oxc_parser/src/lexer/trivia_builder.rs | 41 ++++++++----------- tasks/coverage/codegen_misc.snap | 4 +- tasks/coverage/misc/pass/oxc-2592.ts | 4 ++ tasks/coverage/misc/pass/oxc-2674.tsx | 9 ++++ tasks/coverage/parser_babel.snap | 5 +-- tasks/coverage/parser_misc.snap | 4 +- tasks/coverage/parser_typescript.snap | 6 +-- tasks/coverage/prettier_misc.snap | 5 ++- tasks/coverage/prettier_typescript.snap | 3 +- 10 files changed, 42 insertions(+), 45 deletions(-) create mode 100644 tasks/coverage/misc/pass/oxc-2592.ts create mode 100644 tasks/coverage/misc/pass/oxc-2674.tsx diff --git a/crates/oxc_parser/src/lexer/mod.rs b/crates/oxc_parser/src/lexer/mod.rs index 58853cb53f472..79ba772cccd57 100644 --- a/crates/oxc_parser/src/lexer/mod.rs +++ b/crates/oxc_parser/src/lexer/mod.rs @@ -37,7 +37,7 @@ use oxc_span::{SourceType, Span}; use self::{ byte_handlers::handle_byte, source::{Source, SourcePosition}, - trivia_builder::{TriviaBuilder, TriviasCheckpoint}, + trivia_builder::TriviaBuilder, }; pub use self::{ kind::Kind, @@ -54,8 +54,6 @@ pub struct LexerCheckpoint<'a> { token: Token, errors_pos: usize, - - trivias: TriviasCheckpoint, } #[derive(Debug, Copy, Clone, Eq, PartialEq)] @@ -155,14 +153,12 @@ impl<'a> Lexer<'a> { position: self.source.position(), token: self.token, errors_pos: self.errors.len(), - trivias: self.trivia_builder.checkpoint(), } } /// Rewinds the lexer to the same state as when the passed in `checkpoint` was created. pub fn rewind(&mut self, checkpoint: LexerCheckpoint<'a>) { self.errors.truncate(checkpoint.errors_pos); - self.trivia_builder.rewind(checkpoint.trivias); self.source.set_position(checkpoint.position); self.token = checkpoint.token; self.lookahead.clear(); diff --git a/crates/oxc_parser/src/lexer/trivia_builder.rs b/crates/oxc_parser/src/lexer/trivia_builder.rs index 4b90ad8cc33c5..783e133716b6c 100644 --- a/crates/oxc_parser/src/lexer/trivia_builder.rs +++ b/crates/oxc_parser/src/lexer/trivia_builder.rs @@ -1,45 +1,38 @@ +use std::collections::BTreeMap; + use oxc_ast::{CommentKind, Trivias}; use oxc_span::Span; #[derive(Debug, Default)] pub struct TriviaBuilder { - trivias: Trivias, -} - -#[derive(Debug, Clone, Copy)] -pub struct TriviasCheckpoint { - comments_len: usize, - irregular_whitespaces_len: usize, + // Duplicated comments can be added from rewind, use `BTreeMap` to ensure uniqueness + comments: BTreeMap, + irregular_whitespaces: Vec, } impl TriviaBuilder { pub fn build(self) -> Trivias { - self.trivias - } - - pub fn checkpoint(&self) -> TriviasCheckpoint { - TriviasCheckpoint { - comments_len: self.trivias.comments.len(), - irregular_whitespaces_len: self.trivias.irregular_whitespaces.len(), + Trivias { + comments: self + .comments + .into_iter() + .map(|(start, (end, kind))| (start, end, kind)) + .collect(), + irregular_whitespaces: self.irregular_whitespaces, } } - pub fn rewind(&mut self, checkpoint: TriviasCheckpoint) { - self.trivias.comments.truncate(checkpoint.comments_len); - self.trivias.irregular_whitespaces.truncate(checkpoint.irregular_whitespaces_len); - } - - /// skip leading `//` pub fn add_single_line_comment(&mut self, start: u32, end: u32) { - self.trivias.comments.push((start + 2, end, CommentKind::SingleLine)); + // skip leading `//` + self.comments.insert(start + 2, (end, CommentKind::SingleLine)); } - /// skip leading `/*` and trailing `*/` pub fn add_multi_line_comment(&mut self, start: u32, end: u32) { - self.trivias.comments.push((start + 2, end - 2, CommentKind::MultiLine)); + // skip leading `/*` and trailing `*/` + self.comments.insert(start + 2, (end - 2, CommentKind::MultiLine)); } pub fn add_irregular_whitespace(&mut self, start: u32, end: u32) { - self.trivias.irregular_whitespaces.push(Span::new(start, end)); + self.irregular_whitespaces.push(Span::new(start, end)); } } diff --git a/tasks/coverage/codegen_misc.snap b/tasks/coverage/codegen_misc.snap index b3abb5718cff2..7fe960a68103b 100644 --- a/tasks/coverage/codegen_misc.snap +++ b/tasks/coverage/codegen_misc.snap @@ -1,3 +1,3 @@ codegen_misc Summary: -AST Parsed : 12/12 (100.00%) -Positive Passed: 12/12 (100.00%) +AST Parsed : 14/14 (100.00%) +Positive Passed: 14/14 (100.00%) diff --git a/tasks/coverage/misc/pass/oxc-2592.ts b/tasks/coverage/misc/pass/oxc-2592.ts new file mode 100644 index 0000000000000..c1ebcb1b58d63 --- /dev/null +++ b/tasks/coverage/misc/pass/oxc-2592.ts @@ -0,0 +1,4 @@ +export type Foo = ( + // comment + "bar" +) diff --git a/tasks/coverage/misc/pass/oxc-2674.tsx b/tasks/coverage/misc/pass/oxc-2674.tsx new file mode 100644 index 0000000000000..236cb4ec72dd7 --- /dev/null +++ b/tasks/coverage/misc/pass/oxc-2674.tsx @@ -0,0 +1,9 @@ +const foo =
; + +; + +; diff --git a/tasks/coverage/parser_babel.snap b/tasks/coverage/parser_babel.snap index 1a8f9264dda1a..8b2f6f609be67 100644 --- a/tasks/coverage/parser_babel.snap +++ b/tasks/coverage/parser_babel.snap @@ -1,6 +1,6 @@ parser_babel Summary: AST Parsed : 2090/2096 (99.71%) -Positive Passed: 2080/2096 (99.24%) +Positive Passed: 2083/2096 (99.38%) Negative Passed: 1362/1500 (90.80%) Expect Syntax Error: "annex-b/disabled/1.1-html-comments-close/input.js" Expect Syntax Error: "annex-b/disabled/3.1-sloppy-labeled-functions/input.js" @@ -140,9 +140,6 @@ Expect Syntax Error: "typescript/types/read-only-3/input.ts" Expect Syntax Error: "typescript/types/read-only-4/input.ts" Expect Syntax Error: "typescript/types/tuple-optional-invalid/input.ts" Expect Syntax Error: "typescript/types/tuple-required-after-labeled-optional/input.ts" -Duplicated comments " 3 ": "comments/basic/nested-parentheses/input.js" -Duplicated comments " 1 ": "comments/basic/object-method/input.js" -Duplicated comments " 3 ": "comments/basic/sequence-expression/input.js" Expect to Parse: "core/opts/allowNewTargetOutsideFunction-true/input.js" × Unexpected new.target expression diff --git a/tasks/coverage/parser_misc.snap b/tasks/coverage/parser_misc.snap index 1fdbb3827e13f..4a6c8f05ed3bf 100644 --- a/tasks/coverage/parser_misc.snap +++ b/tasks/coverage/parser_misc.snap @@ -1,6 +1,6 @@ parser_misc Summary: -AST Parsed : 12/12 (100.00%) -Positive Passed: 12/12 (100.00%) +AST Parsed : 14/14 (100.00%) +Positive Passed: 14/14 (100.00%) Negative Passed: 8/8 (100.00%) × Unexpected token diff --git a/tasks/coverage/parser_typescript.snap b/tasks/coverage/parser_typescript.snap index 052782d1ed183..010e4878d2df0 100644 --- a/tasks/coverage/parser_typescript.snap +++ b/tasks/coverage/parser_typescript.snap @@ -1,6 +1,6 @@ parser_typescript Summary: AST Parsed : 5240/5243 (99.94%) -Positive Passed: 5231/5243 (99.77%) +Positive Passed: 5233/5243 (99.81%) Negative Passed: 1062/4879 (21.77%) Expect Syntax Error: "compiler/ClassDeclaration10.ts" Expect Syntax Error: "compiler/ClassDeclaration11.ts" @@ -2697,7 +2697,7 @@ Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardOfFormTypeOfEq Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardOfFormTypeOfNotEqualHasNoEffect.ts" Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardOfFormTypeOfOther.ts" Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsDefeat.ts" -Duplicated comments " change value of x": "conformance/expressions/typeGuards/typeGuardsInIfStatement.ts" +Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsInIfStatement.ts" Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsWithAny.ts" Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsWithInstanceOf.ts" Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsWithInstanceOfByConstructorSignature.ts" @@ -3819,7 +3819,6 @@ Expect Syntax Error: "conformance/types/unknown/unknownControlFlow.ts" Expect Syntax Error: "conformance/types/unknown/unknownType1.ts" Expect Syntax Error: "conformance/types/unknown/unknownType2.ts" Expect Syntax Error: "conformance/types/witness/witness.ts" -Duplicated comments "comment2": "compiler/awaitExpressionInnerCommentEmit.ts" Expect to Parse: "compiler/bom-utf16be.ts" × Invalid Character `￾` @@ -3854,7 +3853,6 @@ Expect to Parse: "compiler/emitBundleWithShebangAndPrologueDirectives1.ts" · ─ 7 │ "use strict" ╰──── -Duplicated comments " must be identifier?": "compiler/jsxNestedWithinTernaryParsesCorrectly.tsx" Expect to Parse: "compiler/withStatementInternalComments.ts" × 'with' statements are not allowed diff --git a/tasks/coverage/prettier_misc.snap b/tasks/coverage/prettier_misc.snap index 7c766bad9cd55..6c052f32d8761 100644 --- a/tasks/coverage/prettier_misc.snap +++ b/tasks/coverage/prettier_misc.snap @@ -1,8 +1,9 @@ prettier_misc Summary: -AST Parsed : 12/12 (100.00%) -Positive Passed: 7/12 (58.33%) +AST Parsed : 14/14 (100.00%) +Positive Passed: 8/14 (57.14%) Expect to Parse: "pass/oxc-1740.tsx" Expect to Parse: "pass/oxc-2087.ts" Expect to Parse: "pass/oxc-2394.ts" +Expect to Parse: "pass/oxc-2674.tsx" Expect to Parse: "pass/swc-1627.js" Expect to Parse: "pass/swc-8243.tsx" diff --git a/tasks/coverage/prettier_typescript.snap b/tasks/coverage/prettier_typescript.snap index ec5ce472d21e8..38f1c8b124726 100644 --- a/tasks/coverage/prettier_typescript.snap +++ b/tasks/coverage/prettier_typescript.snap @@ -1,6 +1,6 @@ prettier_typescript Summary: AST Parsed : 5243/5243 (100.00%) -Positive Passed: 2871/5243 (54.76%) +Positive Passed: 2872/5243 (54.78%) Expect to Parse: "compiler/DeclarationErrorsNoEmitOnError.ts" Expect to Parse: "compiler/abstractInterfaceIdentifierName.ts" Expect to Parse: "compiler/abstractPropertyBasics.ts" @@ -72,7 +72,6 @@ Expect to Parse: "compiler/augmentedTypesModules3b.ts" Expect to Parse: "compiler/augmentedTypesModules4.ts" Expect to Parse: "compiler/autonumberingInEnums.ts" Expect to Parse: "compiler/avoidCycleWithVoidExpressionReturnedFromArrow.ts" -Expect to Parse: "compiler/awaitExpressionInnerCommentEmit.ts" Expect to Parse: "compiler/awaitUnionPromise.ts" Expect to Parse: "compiler/awaitedTypeCrash.ts" Expect to Parse: "compiler/awaitedTypeJQuery.ts"