Trying to fix #668

pydoctor/epydoc/markup/_pyval_repr.py

@@ -52,6 +52,8 @@
 from pydoctor.epydoc.markup.restructuredtext import ParsedRstDocstring
 from pydoctor.epydoc.docutils import set_node_attributes, wbr, obj_reference
 from pydoctor.astutils import node2dottedname, bind_args
+from pydoctor.node2stan import gettext
+
 
 def decode_with_backslashreplace(s: bytes) -> str:
     r"""
@@ -790,9 +792,23 @@ def _colorize_re_pattern(self, pat: AnyStr, state: _ColorizerState, prefix: AnyS
             # By default, do not display the '(?u)'
             self._colorize_re_flags(flags, state)
 
+        marked = state.mark()
+
         # Colorize it!
         self._colorize_re_tree(tree.data, state, True, groups)
 
+        # Our regex understanding is not up to date with python's, we use python 3.6 engine.
+        # This causes some regular expression to be falsely interpreted, so we check if the 
+        # colorized regex round trips, and if it doesn't, then use the regular string colorization.
+        colorized_regex_text = gettext(state.result[marked.length:])
+        if isinstance(expected, bytes):
+            try:
+                colorized_regex_text = bytes(colorized_regex_text, encoding='utf-8')
+            except Exception:
+                raise ValueError("canot encode regex as utf-8")
+        if colorized_regex_text != pat:
+            raise ValueError("regex doesn't round-trips")
+
         # Close quote.
         self._output(quote, self.QUOTE_TAG, state)
 

pydoctor/test/epydoc/test_pyval_repr.py

@@ -1244,9 +1244,10 @@ def color_re(s: Union[bytes, str],
 
         expected = s
         if not raw_string:
-            assert isinstance(expected, str) 
-            # we only test invalid regexes with strings currently
-            expected = expected.replace('\\', '\\\\')
+            if isinstance(expected, bytes):
+                expected = expected.replace(b'\\', b'\\\\')
+            else:
+                expected = expected.replace('\\', '\\\\')
 
         assert round_trip == expected, "%s != %s" % (repr(round_trip), repr(s))
 
@@ -1255,7 +1256,7 @@ def color_re(s: Union[bytes, str],
 
 def test_re_literals() -> None:
     # Literal characters
-    assert color_re(r'abc \t\r\n\f\v \xff \uffff', False) == r"""r<span class="rst-variable-quote">'</span>abc \t\r\n\f\v \xff \uffff<span class="rst-variable-quote">'</span>"""
+    assert color_re(r'abc \t\r\n\f\v \xff \uffff') == r"""r<span class="rst-variable-quote">'</span>abc \t\r\n\f\v \xff \uffff<span class="rst-variable-quote">'</span>"""
 
     assert color_re(r'\.\^\$\\\*\+\?\{\}\[\]\|\(\)\'') == r"""r<span class="rst-variable-quote">'</span>\.\^\$\\\*\+\?\{\}\[\]\|\(\)\'<span class="rst-variable-quote">'</span>"""
 
@@ -1357,7 +1358,7 @@ def test_re_flags() -> None:
 
     assert color_re(r"(?imstux)^Food") == """r<span class="rst-variable-quote">'</span><span class="rst-re-flags">(?imstux)</span>^Food<span class="rst-variable-quote">'</span>"""
 
-    assert color_re(r"(?x)This   is   verbose", False) == """r<span class="rst-variable-quote">'</span><span class="rst-re-flags">(?ux)</span>Thisisverbose<span class="rst-variable-quote">'</span>"""
+    # assert color_re(r"(?x)This   is   verbose") == """r<span class="rst-variable-quote">'</span><span class="rst-re-flags">(?ux)</span>Thisisverbose<span class="rst-variable-quote">'</span>"""
 
 def test_unsupported_regex_features() -> None:
     """
@@ -1369,7 +1370,6 @@ def test_unsupported_regex_features() -> None:
     regexes = ['e*+e',
         '(e?){2,4}+a',
         r"^(\w){1,2}+$",
-        # "^x{}+$", this one fails to round-trip :/
         r'a++',
         r'(?:ab)++',
         r'(?:ab){1,3}+',
@@ -1380,6 +1380,15 @@ def test_unsupported_regex_features() -> None:
     for r in regexes:
         color_re(r)
 
+def test_regex_corner_case_roudtrips() -> None:
+    color_re("^x{}+$")
+    color_re(r"(?x)This   is   verbose")
+    color_re(r'abc \t\r\n\f\v \xff \uffff')
+
+    color_re(b"^x{}+$")
+    color_re(rb"(?x)This   is   verbose")
+    color_re(rb'abc \t\r\n\f\v \xff \uffff')
+
 def test_re_not_literal() -> None:
 
     assert color_re(r"[^0-9]") == """r<span class="rst-variable-quote">'</span><span class="rst-re-group">[</span><span class="rst-re-op">^</span>0<span class="rst-re-op">-</span>9<span class="rst-re-group">]</span><span class="rst-variable-quote">'</span>"""