From b677a643c5322b6af2dbca1f8f7a4f8c98e732aa Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sun, 2 Jun 2024 01:31:36 +0530 Subject: [PATCH] tokenizer: skip lines that are just slash and whitespace (#4343) --- AUTHORS.md | 1 + CHANGES.md | 2 ++ src/blib2to3/pgen2/tokenize.py | 6 +++++ tests/data/cases/backslash_before_indent.py | 24 +++++++++++++++++++ .../cases/comment_after_escaped_newline.py | 4 +++- tests/data/cases/form_feeds.py | 1 + 6 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/data/cases/backslash_before_indent.py diff --git a/AUTHORS.md b/AUTHORS.md index e0511bb9b7c..fa1ce8f7112 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -181,6 +181,7 @@ Multiple contributions by: - [Tony Narlock](mailto:tony@git-pull.com) - [Tsuyoshi Hombashi](mailto:tsuyoshi.hombashi@gmail.com) - [Tushar Chandra](mailto:tusharchandra2018@u.northwestern.edu) +- [Tushar Sadhwani](mailto:tushar.sadhwani000@gmail.com) - [Tzu-ping Chung](mailto:uranusjr@gmail.com) - [Utsav Shah](mailto:ukshah2@illinois.edu) - utsav-dbx diff --git a/CHANGES.md b/CHANGES.md index 68a2fe35205..a63503d539a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,8 @@ - Fix regression where Black failed to parse a multiline f-string containing another multiline string (#4339) +- Fix bug with Black incorrectly parsing empty lines with a backslash (#4343) + ### Performance diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index b86b91ed37f..28972a9bd78 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -608,6 +608,12 @@ def generate_tokens( except StopIteration: line = "" lnum += 1 + + # skip lines that are just indent characters ending with a slash + # to avoid storing that line's indent information. + if not contstr and line.rstrip("\n").strip(" \t\f") == "\\": + continue + pos, max = 0, len(line) if contstr: # continued string diff --git a/tests/data/cases/backslash_before_indent.py b/tests/data/cases/backslash_before_indent.py new file mode 100644 index 00000000000..2d126a945e4 --- /dev/null +++ b/tests/data/cases/backslash_before_indent.py @@ -0,0 +1,24 @@ +# flags: --minimum-version=3.10 +class Plotter: +\ + pass + +class AnotherCase: + \ + """Some + \ + Docstring + """ + +# output + +class Plotter: + + pass + + +class AnotherCase: + """Some + \ + Docstring + """ diff --git a/tests/data/cases/comment_after_escaped_newline.py b/tests/data/cases/comment_after_escaped_newline.py index 133f4898a47..133c230a3a8 100644 --- a/tests/data/cases/comment_after_escaped_newline.py +++ b/tests/data/cases/comment_after_escaped_newline.py @@ -14,5 +14,7 @@ def bob(): # pylint: disable=W9016 pass -def bobtwo(): # some comment here +def bobtwo(): + + # some comment here pass diff --git a/tests/data/cases/form_feeds.py b/tests/data/cases/form_feeds.py index 48ffc98106b..957b4a1db95 100644 --- a/tests/data/cases/form_feeds.py +++ b/tests/data/cases/form_feeds.py @@ -156,6 +156,7 @@ def something(self): # + # pass