From ba02e6fe81bb12efff3f7fb584adaa4a2b387ecd Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 11 Dec 2024 13:54:49 -0500 Subject: [PATCH] Called wrong check routine --- mathics_scanner/feed.py | 6 ++++-- mathics_scanner/tokeniser.py | 18 ++++++++---------- test/test_string_tokens.py | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/mathics_scanner/feed.py b/mathics_scanner/feed.py index 8cc0d8f..b31438c 100644 --- a/mathics_scanner/feed.py +++ b/mathics_scanner/feed.py @@ -47,7 +47,8 @@ def empty(self) -> bool: def message(self, symbol_name: str, tag: str, *args) -> None: """ - A Generic message appending routine. the ``self.messages`` message queue. + A Generic routine for appending a message to the ``self.messages`` message + queue. ``symbol_name`` is usually the string symbol name of the built-in function that is recording the error. "Syntax" error is the exception to this rule. @@ -63,6 +64,7 @@ def message(self, symbol_name: str, tag: str, *args) -> None: "Part" is the symbol_name, "partw" is the tag and args is: (,)>, ) + """ if symbol_name == "Syntax": @@ -74,7 +76,7 @@ def message(self, symbol_name: str, tag: str, *args) -> None: def syntax_message(self, symbol_name: str, tag: str, *args) -> list: """ - Append a syntax-message error message to the message queue. + Append a "Syntax" error message to the message queue. """ if len(args) > 3: diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index 90a41f4..11638e4 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -414,19 +414,17 @@ def incomplete(self): self.prescanner.incomplete() self.code += self.prescanner.replace_escape_sequences() - def sntx_message(self, pos: Optional[int] = None, tag: Optional[str] = None): + def sntx_message(self, pos: Optional[int] = None): """ Send a "syntx{b,f} error message to the input-reading feeder. """ if pos is None: pos = self.pos pre, post = self.code[:pos], self.code[pos:].rstrip("\n") - if tag is None: - tag = "sntxb" if pos == 0 else "sntxf" if pos == 0: - self.feeder.message("Syntax", tag, post) + self.feeder.message("Syntax", "sntxb", post) else: - self.feeder.message("Syntax", tag, pre, post) + self.feeder.message("Syntax", "sntxf", pre, post) # TODO: Convert this to __next__ in the future. def next(self) -> Token: @@ -551,13 +549,13 @@ def t_String(self, match: re.Match) -> Token: "Break out from self.code the next token which is expected to be a String" start, end = self.pos, None self.pos += 1 # skip opening '"' - skipped_chars = [] + newlines = [] while True: if self.pos >= len(self.code): if end is None: # reached end while still inside string self.incomplete() - skipped_chars.append(self.pos) + newlines.append(self.pos) else: break char = self.code[self.pos] @@ -573,7 +571,7 @@ def t_String(self, match: re.Match) -> Token: if self.pos + 1 == len(self.code): # We have a \ at the end of a line. self.incomplete() - skipped_chars.append(self.pos) + newlines.append(self.pos) # Code below is in pre-scanner. We might decide # later to move that code here. @@ -586,7 +584,7 @@ def t_String(self, match: re.Match) -> Token: # "\\" have the backslash preserved. But for other # characters, the backslash is removed. if self.code[self.pos + 1] not in ( - "b", # bell? + "b", # word boundary? "f", # form-feed? "n", # newline "r", # carrage return @@ -604,7 +602,7 @@ def t_String(self, match: re.Match) -> Token: else: self.pos += 1 - indices = [start] + skipped_chars + [end] + indices = [start] + newlines + [end] result = "".join( self.code[indices[i] : indices[i + 1]] for i in range(len(indices) - 1) ) diff --git a/test/test_string_tokens.py b/test/test_string_tokens.py index 9e73a55..092905a 100644 --- a/test/test_string_tokens.py +++ b/test/test_string_tokens.py @@ -51,8 +51,8 @@ def get_tokens(source_text: str): def test_string(): - for ctrl_char in ("\b", "\f", "\n", "\r", "\t"): - check_string(f'"a{ctrl_char}"', f'"a{ctrl_char}"') + for escape_string in ("\b", "\f", "\n", "\r", "\t"): + check_string(f'"a{escape_string}"', f'"a{escape_string}"') # Broken: # "a\050", "a\051" "a\052" @@ -64,4 +64,4 @@ def test_string(): check_string(r'"a\"b\\c"', r'"a\"b\\c"') incomplete_error(r'"abc', "String does not have terminating quote") incomplete_error(r'"\"', "Unterminated escape sequence") - incomplete_error(r'"a\X"', '"X" is not a valid escape character') + scan_error(r'"a\X"', '"X" is not a valid escape character')