Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf(parser): reduce Token size to 8 bytes from 16 #8153

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
+229 −122
Draft

perf(parser): reduce Token size to 8 bytes from 16 #8153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ab62c23
replace token.end with token.len and pack bools
branchseer Dec 28, 2024
c3d0ec3
update token usages
branchseer Dec 28, 2024
557c1be
cargo fmt
branchseer Dec 28, 2024
93f4f1a
fix cargo lint warnings
branchseer Dec 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions crates/oxc_parser/src/cursor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ impl<'a> ParserImpl<'a> {

/// Get current source text
pub(crate) fn cur_src(&self) -> &'a str {
let range = self.cur_token().span();
let range = self.lexer.token_span(self.cur_token());
// SAFETY:
// range comes from the lexer, which are ensured to meeting the criteria of `get_unchecked`.

Expand Down Expand Up @@ -109,13 +109,23 @@ impl<'a> ParserImpl<'a> {
self.cur_kind() == kind
}

#[inline]
pub(crate) fn cur_token_span(&self) -> Span {
self.lexer.token_span(self.cur_token())
}

#[inline]
pub(crate) fn cur_token_end(&self) -> u32 {
self.lexer.token_end(self.cur_token())
}

/// `StringValue` of `IdentifierName` normalizes any Unicode escape sequences
/// in `IdentifierName` hence such escapes cannot be used to write an Identifier
/// whose code point sequence is the same as a `ReservedWord`.
#[inline]
fn test_escaped_keyword(&mut self, kind: Kind) {
if self.cur_token().escaped() && kind.is_all_keyword() {
let span = self.cur_token().span();
let span = self.cur_token_span();
self.error(diagnostics::escaped_keyword(span));
}
}
Expand All @@ -124,15 +134,15 @@ impl<'a> ParserImpl<'a> {
/// Checks if the current token is escaped if it is a keyword
fn advance(&mut self, kind: Kind) {
self.test_escaped_keyword(kind);
self.prev_token_end = self.token.end;
self.prev_token_end = self.lexer.token_end(self.token);
self.token = self.lexer.next_token();
}

/// Move to the next `JSXChild`
/// Checks if the current token is escaped if it is a keyword
fn advance_for_jsx_child(&mut self, kind: Kind) {
self.test_escaped_keyword(kind);
self.prev_token_end = self.token.end;
self.prev_token_end = self.lexer.token_end(self.token);
self.token = self.lexer.next_jsx_child();
}

Expand Down Expand Up @@ -184,13 +194,13 @@ impl<'a> ParserImpl<'a> {
if kind == Kind::Semicolon {
return true;
}
kind == Kind::RCurly || kind.is_eof() || self.cur_token().is_on_new_line
kind == Kind::RCurly || kind.is_eof() || self.cur_token().is_on_new_line()
}

/// # Errors
pub(crate) fn expect_without_advance(&mut self, kind: Kind) -> Result<()> {
if !self.at(kind) {
let range = self.cur_token().span();
let range = self.cur_token_span();
return Err(diagnostics::expect_token(kind.to_str(), self.cur_kind().to_str(), range));
}
Ok(())
Expand Down Expand Up @@ -238,7 +248,7 @@ impl<'a> ParserImpl<'a> {

/// Tell lexer to continue reading jsx identifier if the lexer character position is at `-` for `<component-name>`
pub(crate) fn continue_lex_jsx_identifier(&mut self) {
if let Some(token) = self.lexer.continue_lex_jsx_identifier() {
if let Some(token) = self.lexer.continue_lex_jsx_identifier(self.token.start) {
self.token = token;
}
}
Expand Down
12 changes: 6 additions & 6 deletions crates/oxc_parser/src/js/arrow.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ impl<'a> ParserImpl<'a> {
if self.at(Kind::Async) {
let second_token = self.peek_token();
let second = second_token.kind;
if second_token.is_on_new_line {
if second_token.is_on_new_line() {
return Tristate::False;
}
if second != Kind::LParen && second != Kind::LAngle {
Expand Down Expand Up @@ -185,7 +185,7 @@ impl<'a> ParserImpl<'a> {
let first = first_token.kind;
// If the "async" is followed by "=>" token then it is not a beginning of an async arrow-function
// but instead a simple arrow-function which will be parsed inside "parseAssignmentExpressionOrHigher"
if first_token.is_on_new_line || first == Kind::Arrow {
if first_token.is_on_new_line() || first == Kind::Arrow {
return Tristate::False;
}
// Check for un-parenthesized AsyncArrowFunction
Expand Down Expand Up @@ -230,8 +230,8 @@ impl<'a> ParserImpl<'a> {

self.ctx = self.ctx.and_await(has_await);

if self.cur_token().is_on_new_line {
self.error(diagnostics::lineterminator_before_arrow(self.cur_token().span()));
if self.cur_token().is_on_new_line() {
self.error(diagnostics::lineterminator_before_arrow(self.cur_token_span()));
}

self.expect(Kind::Arrow)?;
Expand Down Expand Up @@ -262,8 +262,8 @@ impl<'a> ParserImpl<'a> {

self.ctx = self.ctx.and_await(has_await);

if self.cur_token().is_on_new_line {
self.error(diagnostics::lineterminator_before_arrow(self.cur_token().span()));
if self.cur_token().is_on_new_line() {
self.error(diagnostics::lineterminator_before_arrow(self.cur_token_span()));
}

self.expect(Kind::Arrow)?;
Expand Down
4 changes: 2 additions & 2 deletions crates/oxc_parser/src/js/binding.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ impl<'a> ParserImpl<'a> {
let elem = self.parse_rest_element()?;
if self.at(Kind::Comma) {
if matches!(self.peek_kind(), Kind::RCurly | Kind::RBrack) {
let span = self.cur_token().span();
let span = self.cur_token_span();
self.bump_any();
self.error(diagnostics::binding_rest_element_trailing_comma(span));
}
Expand All @@ -111,7 +111,7 @@ impl<'a> ParserImpl<'a> {
let kind = self.parse_binding_pattern_kind()?;
// Rest element does not allow `?`, checked in checker/typescript.rs
if self.at(Kind::Question) && self.is_ts {
let span = self.cur_token().span();
let span = self.cur_token_span();
self.bump_any();
self.error(diagnostics::a_rest_parameter_cannot_be_optional(span));
}
Expand Down
2 changes: 1 addition & 1 deletion crates/oxc_parser/src/js/class.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ impl<'a> ParserImpl<'a> {

// async ...
if key_name.is_none() && self.at(Kind::Async) && !self.peek_at(Kind::Question) {
if !self.peek_token().is_on_new_line
if !self.peek_token().is_on_new_line()
&& (self.peek_kind().is_class_element_name_start() || self.peek_at(Kind::Star))
{
self.bump(Kind::Async);
Expand Down
8 changes: 4 additions & 4 deletions crates/oxc_parser/src/js/declaration.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ impl<'a> ParserImpl<'a> {
let mut definite = false;
if binding_kind.is_binding_identifier()
&& self.at(Kind::Bang)
&& !self.cur_token().is_on_new_line
&& !self.cur_token().is_on_new_line()
{
self.eat(Kind::Bang);
definite = true;
Expand Down Expand Up @@ -152,15 +152,15 @@ impl<'a> ParserImpl<'a> {
self.expect(Kind::Using)?;

// `[no LineTerminator here]`
if self.cur_token().is_on_new_line {
if self.cur_token().is_on_new_line() {
self.error(diagnostics::line_terminator_before_using_declaration(
self.cur_token().span(),
self.cur_token_span(),
));
}

// [lookahead ≠ await]
if self.cur_kind() == Kind::Await {
self.error(diagnostics::await_in_using_declaration(self.cur_token().span()));
self.error(diagnostics::await_in_using_declaration(self.cur_token_span()));
self.eat(Kind::Await);
}

Expand Down
22 changes: 11 additions & 11 deletions crates/oxc_parser/src/js/expression.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ impl<'a> ParserImpl<'a> {
let cur = self.cur_kind();
if !cur.is_binding_identifier() {
let err = if cur.is_reserved_keyword() {
diagnostics::identifier_reserved_word(self.cur_token().span(), cur.to_str())
diagnostics::identifier_reserved_word(self.cur_token_span(), cur.to_str())
} else {
self.unexpected()
};
Expand Down Expand Up @@ -296,7 +296,7 @@ impl<'a> ParserImpl<'a> {
}
_ => unreachable!(),
}
.map_err(|err| diagnostics::invalid_number(err, token.span()))?;
.map_err(|err| diagnostics::invalid_number(err, self.cur_token_span()))?;
let base = match token.kind {
Kind::Decimal => NumberBase::Decimal,
Kind::Float => NumberBase::Float,
Expand Down Expand Up @@ -329,7 +329,7 @@ impl<'a> ParserImpl<'a> {
let raw = self.cur_src();
let src = raw.strip_suffix('n').unwrap();
let _value = parse_big_int(src, token.kind, token.has_separator())
.map_err(|err| diagnostics::invalid_number(err, token.span()))?;
.map_err(|err| diagnostics::invalid_number(err, self.cur_token_span()))?;
self.bump_any();
Ok(self.ast.big_int_literal(self.end_span(span), raw, base))
}
Expand All @@ -341,7 +341,7 @@ impl<'a> ParserImpl<'a> {
let pattern_start = self.cur_token().start + 1; // +1 to exclude left `/`
let pattern_text = &self.source_text[pattern_start as usize..pattern_end as usize];
let flags_start = pattern_end + 1; // +1 to include right `/`
let flags_text = &self.source_text[flags_start as usize..self.cur_token().end as usize];
let flags_text = &self.source_text[flags_start as usize..self.cur_token_end() as usize];
let raw = self.cur_src();
self.bump_any();
// Parse pattern if options is enabled and also flags are valid
Expand Down Expand Up @@ -440,7 +440,7 @@ impl<'a> ParserImpl<'a> {
/// ,
/// Elision ,
pub(crate) fn parse_elision(&mut self) -> ArrayExpressionElement<'a> {
self.ast.array_expression_element_elision(self.cur_token().span())
self.ast.array_expression_element_elision(self.cur_token_span())
}

/// Section [Template Literal](https://tc39.es/ecma262/#prod-TemplateLiteral)
Expand Down Expand Up @@ -689,7 +689,7 @@ impl<'a> ParserImpl<'a> {
Kind::LBrack if !self.ctx.has_decorator() => {
self.parse_computed_member_expression(lhs_span, lhs, false)?
}
Kind::Bang if !self.cur_token().is_on_new_line && self.is_ts => {
Kind::Bang if !self.cur_token().is_on_new_line() && self.is_ts => {
self.bump_any();
self.ast.expression_ts_non_null(self.end_span(lhs_span), lhs)
}
Expand Down Expand Up @@ -926,7 +926,7 @@ impl<'a> ParserImpl<'a> {
let span = self.start_span();
let lhs = self.parse_lhs_expression_or_higher()?;
// ++ -- postfix update expressions
if self.cur_kind().is_update_operator() && !self.cur_token().is_on_new_line {
if self.cur_kind().is_update_operator() && !self.cur_token().is_on_new_line() {
let operator = map_update_operator(self.cur_kind());
self.bump_any();
let lhs = SimpleAssignmentTarget::cover(lhs, self)?;
Expand Down Expand Up @@ -1028,7 +1028,7 @@ impl<'a> ParserImpl<'a> {
}

if self.is_ts && matches!(kind, Kind::As | Kind::Satisfies) {
if self.cur_token().is_on_new_line {
if self.cur_token().is_on_new_line() {
break;
}
self.bump_any();
Expand Down Expand Up @@ -1157,7 +1157,7 @@ impl<'a> ParserImpl<'a> {
fn parse_await_expression(&mut self, lhs_span: Span) -> Result<Expression<'a>> {
let span = self.start_span();
if !self.ctx.has_await() {
self.error(diagnostics::await_expression(self.cur_token().span()));
self.error(diagnostics::await_expression(self.cur_token_span()));
}
self.bump_any();
let argument = self.context(Context::Await, Context::empty(), |p| {
Expand Down Expand Up @@ -1214,7 +1214,7 @@ impl<'a> ParserImpl<'a> {
return false;
}

return !peek_token.is_on_new_line && peek_token.kind.is_after_await_or_yield();
return !peek_token.is_on_new_line() && peek_token.kind.is_after_await_or_yield();
}
false
}
Expand All @@ -1229,7 +1229,7 @@ impl<'a> ParserImpl<'a> {
if self.ctx.has_yield() {
return true;
}
return !peek_token.is_on_new_line && peek_token.kind.is_after_await_or_yield();
return !peek_token.is_on_new_line() && peek_token.kind.is_after_await_or_yield();
}
false
}
Expand Down
8 changes: 4 additions & 4 deletions crates/oxc_parser/src/js/function.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ impl<'a> ParserImpl<'a> {
self.at(Kind::Function)
|| self.at(Kind::Async)
&& self.peek_at(Kind::Function)
&& !self.peek_token().is_on_new_line
&& !self.peek_token().is_on_new_line()
}

pub(crate) fn parse_function_body(&mut self) -> Result<Box<'a, FunctionBody<'a>>> {
Expand Down Expand Up @@ -99,7 +99,7 @@ impl<'a> ParserImpl<'a> {
let element = self.parse_rest_element()?;
if self.at(Kind::Comma) {
if matches!(self.peek_kind(), Kind::RCurly | Kind::RBrack) {
let span = self.cur_token().span();
let span = self.cur_token_span();
self.bump_any();
self.error(diagnostics::binding_rest_element_trailing_comma(span));
}
Expand Down Expand Up @@ -293,7 +293,7 @@ impl<'a> ParserImpl<'a> {
let mut delegate = false;
let mut argument = None;

if !self.cur_token().is_on_new_line {
if !self.cur_token().is_on_new_line() {
delegate = self.eat(Kind::Star);
let not_assignment_expr = matches!(
self.cur_kind(),
Expand Down Expand Up @@ -336,7 +336,7 @@ impl<'a> ParserImpl<'a> {
if kind.is_id_required() && id.is_none() {
match self.cur_kind() {
Kind::LParen => {
self.error(diagnostics::expect_function_name(self.cur_token().span()));
self.error(diagnostics::expect_function_name(self.cur_token_span()));
}
kind if kind.is_reserved_keyword() => self.expect_without_advance(Kind::Ident)?,
_ => {}
Expand Down
4 changes: 2 additions & 2 deletions crates/oxc_parser/src/js/module.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ impl<'a> ParserImpl<'a> {
/// [Import Attributes](https://tc39.es/proposal-import-attributes)
fn parse_import_attributes(&mut self) -> Result<Option<WithClause<'a>>> {
let attributes_keyword = match self.cur_kind() {
Kind::Assert if !self.cur_token().is_on_new_line => self.parse_identifier_name()?,
Kind::Assert if !self.cur_token().is_on_new_line() => self.parse_identifier_name()?,
Kind::With => self.parse_identifier_name()?,
_ => {
return Ok(None);
Expand Down Expand Up @@ -391,7 +391,7 @@ impl<'a> ParserImpl<'a> {
self.parse_class_declaration(decl_span, &modifiers)
.map(ExportDefaultDeclarationKind::ClassDeclaration)?
}
_ if self.at(Kind::Interface) && !self.peek_token().is_on_new_line && self.is_ts => {
_ if self.at(Kind::Interface) && !self.peek_token().is_on_new_line() && self.is_ts => {
self.parse_ts_interface_declaration(decl_span, &Modifiers::empty()).map(|decl| {
match decl {
Declaration::TSInterfaceDeclaration(decl) => {
Expand Down
4 changes: 2 additions & 2 deletions crates/oxc_parser/src/js/object.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ impl<'a> ParserImpl<'a> {
// AsyncGeneratorMethod
Kind::Async
if (class_element_name || peek_kind == Kind::Star)
&& !self.peek_token().is_on_new_line =>
&& !self.peek_token().is_on_new_line() =>
{
self.parse_property_definition_method()
}
Expand All @@ -69,7 +69,7 @@ impl<'a> ParserImpl<'a> {
&& modifier_kind.is_modifier_kind()
&& peek_kind.is_identifier_or_keyword() =>
{
if let Ok(modifier) = Modifier::try_from(self.cur_token()) {
if let Ok(modifier) = Modifier::try_from_token(self.cur_token(), &self.lexer) {
self.error(diagnostics::modifier_cannot_be_used_here(&modifier));
} else {
#[cfg(debug_assertions)]
Expand Down
8 changes: 4 additions & 4 deletions crates/oxc_parser/src/js/statement.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ impl<'a> ParserImpl<'a> {
self.parse_using()
}
Kind::Using if self.peek_kind().is_binding_identifier() => self.parse_using(),
Kind::Async if self.peek_at(Kind::Function) && !self.peek_token().is_on_new_line => {
Kind::Async if self.peek_at(Kind::Function) && !self.peek_token().is_on_new_line() => {
self.parse_function_declaration(stmt_ctx)
}
_ if self.is_ts && self.at_start_of_ts_declaration() => {
Expand Down Expand Up @@ -243,7 +243,7 @@ impl<'a> ParserImpl<'a> {
// [+Await]
let r#await = if self.at(Kind::Await) {
if !self.ctx.has_await() {
self.error(diagnostics::await_expression(self.cur_token().span()));
self.error(diagnostics::await_expression(self.cur_token_span()));
}
self.bump_any();
true
Expand Down Expand Up @@ -492,11 +492,11 @@ impl<'a> ParserImpl<'a> {
fn parse_throw_statement(&mut self) -> Result<Statement<'a>> {
let span = self.start_span();
self.bump_any(); // advance `throw`
if self.cur_token().is_on_new_line {
if self.cur_token().is_on_new_line() {
self.error(diagnostics::illegal_newline(
"throw",
self.end_span(span),
self.cur_token().span(),
self.cur_token_span(),
));
}
let argument = self.parse_expr()?;
Expand Down
Loading
Loading