From 86be7ff32f79b7b34bc94ce6c2a18a7db74ccd9c Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Mon, 25 Nov 2024 10:45:09 -0300
Subject: [PATCH 1/2] fix #94

---
 mathics_scanner/tokeniser.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py
index 2559599..f276d3e 100644
--- a/mathics_scanner/tokeniser.py
+++ b/mathics_scanner/tokeniser.py
@@ -470,12 +470,29 @@ def _skip_blank(self):
             if self.pos >= len(self.code):
                 if comment:
                     try:
+                        # Since we are inside a comment,
+                        # we can just do
+                        # ```
+                        # self.prescanner.incomplete()
+                        # self.code = self.prescanner.input_line
+                        # ```
+                        # avoiding the call to the method
+                        # `Prescanner.replace_escape_sequences()`
+                        # which have issues with escape characters...
                         self.incomplete()
                     except ValueError:
-                        # Funny symbols like | in comments can cause a ValueError.
-                        # Until we have a better fix -- like noting we are inside a
-                        # comment and should not try to substitute symbols -- ignore.
-                        pass
+                        # `incomplete` tries to parse substrings like `\|AAAAA`
+                        # that can be interpreted as a character reference.
+                        # To do that, it tries to get the
+                        # new line using the method
+                        # `Prescanner.replace_escape_sequences()`
+                        # Inside a comment, the special meaning of escape sequences
+                        # like `\|` should not be taken into account.
+                        #
+                        # In case of error, just let's pick the code
+                        # from the `input_line` attribute of
+                        # prescanner:
+                        self.code = self.prescanner.input_line
                 else:
                     break
             if comment:

From c251ee3e559935a65b8811cad666abd61e00c6b5 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Mon, 25 Nov 2024 12:28:44 -0300
Subject: [PATCH 2/2] fixing the comment

---
 mathics_scanner/tokeniser.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py
index f276d3e..442b434 100644
--- a/mathics_scanner/tokeniser.py
+++ b/mathics_scanner/tokeniser.py
@@ -470,15 +470,6 @@ def _skip_blank(self):
             if self.pos >= len(self.code):
                 if comment:
                     try:
-                        # Since we are inside a comment,
-                        # we can just do
-                        # ```
-                        # self.prescanner.incomplete()
-                        # self.code = self.prescanner.input_line
-                        # ```
-                        # avoiding the call to the method
-                        # `Prescanner.replace_escape_sequences()`
-                        # which have issues with escape characters...
                         self.incomplete()
                     except ValueError:
                         # `incomplete` tries to parse substrings like `\|AAAAA`
@@ -493,6 +484,8 @@ def _skip_blank(self):
                         # from the `input_line` attribute of
                         # prescanner:
                         self.code = self.prescanner.input_line
+                        # TODO: handle the corner case where the rest of the line
+                        # include escaped sequences, out of the comment.
                 else:
                     break
             if comment: