Markdown code upgrade

funnel/utils/markdown/base.py

@@ -9,6 +9,7 @@
     ClassVar,
     Dict,
     Iterable,
+    Mapping,
     Optional,
     Set,
     Union,
@@ -49,8 +50,6 @@
 
 # --- Markdown dataclasses -------------------------------------------------------------
 
-OptionStrings = Literal['html', 'breaks', 'linkify', 'typographer']
-
 
 @dataclass
 class MarkdownPlugin:
@@ -89,7 +88,7 @@ class MarkdownConfig:
         'default', 'zero', 'commonmark', 'js-default', 'gfm-like'
     ] = 'commonmark'
     #: Updated options against the preset
-    options_update: Optional[Dict[OptionStrings, bool]] = None
+    options_update: Optional[Mapping] = None
     #: Allow only inline rules (skips all block rules)?
     inline: bool = False
 

funnel/utils/markdown/mdit_plugins/abbr.py

@@ -10,7 +10,7 @@
 
 from markdown_it import MarkdownIt
 from markdown_it.rules_block import StateBlock
-from markdown_it.rules_inline import StateInline
+from markdown_it.rules_core import StateCore
 from markdown_it.token import Token
 
 __all__ = ['abbr_plugin']
@@ -57,7 +57,7 @@ def abbr_def(state: StateBlock, start_line: int, end_line: int, silent: bool) ->
     return True
 
 
-def abbr_replace(state: StateInline) -> None:
+def abbr_replace(state: StateCore) -> None:
     """Tokenizes and tags defined abbreviations in content."""
     block_tokens = state.tokens
 

funnel/utils/markdown/mdit_plugins/embeds.py

@@ -67,7 +67,7 @@ def embeds_func(
 
         # Check out the first character quickly,
         # this should filter out most of non-containers
-        if marker_char != state.srcCharCode[start]:
+        if marker_char != ord(state.src[start]):
             return False
 
         # Check out the rest of the marker string
@@ -115,7 +115,7 @@ def embeds_func(
                 #  test
                 break
 
-            if marker_char != state.srcCharCode[start]:
+            if marker_char != ord(state.src[start]):
                 continue
 
             if state.sCount[next_line] - state.blkIndent >= 4:

funnel/utils/markdown/mdit_plugins/heading_anchors_fix.py

@@ -3,12 +3,12 @@
 from __future__ import annotations
 
 from markdown_it import MarkdownIt
-from markdown_it.rules_inline import StateInline
+from markdown_it.rules_core import StateCore
 
 __all__ = ['heading_anchors_fix_plugin']
 
 
-def heading_anchors_fix(state: StateInline) -> None:
+def heading_anchors_fix(state: StateCore) -> None:
     prev_token = None
 
     for token in state.tokens:

funnel/utils/markdown/mdit_plugins/ins_tag.py

@@ -17,19 +17,18 @@
 
 __all__ = ['ins_plugin']
 
-PLUS_CHAR = 0x2B  # ASCII value for `+`
+PLUS_CHAR = '+'
 
 
 def tokenize(state: StateInline, silent: bool) -> bool:
     """Insert each marker as a separate text token, and add it to delimiter list."""
     start = state.pos
-    marker = state.srcCharCode[start]
-    ch = chr(marker)
+    ch = state.src[start]
 
     if silent:
         return False
 
-    if marker != PLUS_CHAR:
+    if ch != PLUS_CHAR:
         return False
 
     scanned = state.scanDelims(state.pos, True)
@@ -50,9 +49,8 @@ def tokenize(state: StateInline, silent: bool) -> bool:
         token.content = ch + ch
         state.delimiters.append(
             Delimiter(
-                marker=marker,
+                marker=ord(ch),
                 length=0,  # disable "rule of 3" length checks meant for emphasis
-                jump=i // 2,  # for `++` 1 marker = 2 characters
                 token=len(state.tokens) - 1,
                 end=-1,
                 open=scanned.can_open,
@@ -71,7 +69,7 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None:
 
     for i in range(0, maximum):
         start_delim = delimiters[i]
-        if start_delim.marker != PLUS_CHAR:
+        if start_delim.marker != ord(PLUS_CHAR):
             i += 1
             continue
 
@@ -85,19 +83,19 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None:
         token.type = 'ins_open'
         token.tag = 'ins'
         token.nesting = 1
-        token.markup = '++'
+        token.markup = PLUS_CHAR * 2
         token.content = ''
 
         token = state.tokens[end_delim.token]
         token.type = 'ins_close'
         token.tag = 'ins'
         token.nesting = -1
-        token.markup = '++'
+        token.markup = PLUS_CHAR * 2
         token.content = ''
 
         end_token = state.tokens[end_delim.token - 1]
 
-        if end_token.type == 'text' and end_token == '+':  # nosec
+        if end_token.type == 'text' and end_token == PLUS_CHAR:  # nosec
             lone_markers.append(end_delim.token - 1)
 
     # If a marker sequence has an odd number of characters, it's split

funnel/utils/markdown/mdit_plugins/mark_tag.py

@@ -17,19 +17,18 @@
 
 __all__ = ['mark_plugin']
 
-EQUALS_CHAR = 0x3D  # ASCII value for `=`
+EQUALS_CHAR = '='
 
 
 def tokenize(state: StateInline, silent: bool) -> bool:
     """Insert each marker as a separate text token, and add it to delimiter list."""
     start = state.pos
-    marker = state.srcCharCode[start]
-    ch = chr(marker)
+    ch = state.src[start]
 
     if silent:
         return False
 
-    if marker != EQUALS_CHAR:
+    if ch != EQUALS_CHAR:
         return False
 
     scanned = state.scanDelims(state.pos, True)
@@ -50,9 +49,8 @@ def tokenize(state: StateInline, silent: bool) -> bool:
         token.content = ch + ch
         state.delimiters.append(
             Delimiter(
-                marker=marker,
+                marker=ord(ch),
                 length=0,  # disable "rule of 3" length checks meant for emphasis
-                jump=i // 2,  # for `==` 1 marker = 2 characters
                 token=len(state.tokens) - 1,
                 end=-1,
                 open=scanned.can_open,
@@ -71,7 +69,7 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None:
 
     for i in range(0, maximum):
         start_delim = delimiters[i]
-        if start_delim.marker != EQUALS_CHAR:
+        if start_delim.marker != ord(EQUALS_CHAR):
             i += 1
             continue
 
@@ -85,19 +83,19 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None:
         token.type = 'mark_open'
         token.tag = 'mark'
         token.nesting = 1
-        token.markup = '=='
+        token.markup = EQUALS_CHAR * 2
         token.content = ''
 
         token = state.tokens[end_delim.token]
         token.type = 'mark_close'
         token.tag = 'mark'
         token.nesting = -1
-        token.markup = '=='
+        token.markup = EQUALS_CHAR * 2
         token.content = ''
 
         end_token = state.tokens[end_delim.token - 1]
 
-        if end_token.type == 'text' and end_token == '=':  # nosec
+        if end_token.type == 'text' and end_token == EQUALS_CHAR:  # nosec
             lone_markers.append(end_delim.token - 1)
 
     # If a marker sequence has an odd number of characters, it's split

funnel/utils/markdown/mdit_plugins/sub_tag.py

@@ -17,22 +17,22 @@
 
 __all__ = ['sub_plugin']
 
-TILDE_CHAR = 0x7E  # ASCII value for `~`
+TILDE_CHAR = '~'
 
 WHITESPACE_RE = re.compile(r'(^|[^\\])(\\\\)*\s')
 UNESCAPE_RE = re.compile(r'\\([ \\!"#$%&\'()*+,.\/:;<=>?@[\]^_`{|}~-])')
 
 
 def tokenize(state: StateInline, silent: bool) -> bool:
     start = state.pos
-    marker = state.srcCharCode[start]
+    ch = state.src[start]
     maximum = state.posMax
     found = False
 
     if silent:
         return False
 
-    if marker != TILDE_CHAR:
+    if ch != TILDE_CHAR:
         return False
 
     # Don't run any pairs in validation mode
@@ -42,7 +42,7 @@ def tokenize(state: StateInline, silent: bool) -> bool:
     state.pos = start + 1
 
     while state.pos < maximum:
-        if state.srcCharCode[state.pos] == TILDE_CHAR:
+        if state.src[state.pos] == TILDE_CHAR:
             found = True
             break
         state.md.inline.skipToken(state)
@@ -63,13 +63,13 @@ def tokenize(state: StateInline, silent: bool) -> bool:
 
     # Earlier we checked "not silent", but this implementation does not need it
     token = state.push('sub_open', 'sub', 1)
-    token.markup = '~'
+    token.markup = TILDE_CHAR
 
     token = state.push('text', '', 0)
     token.content = UNESCAPE_RE.sub('$1', content)
 
     token = state.push('sub_close', 'sub', -1)
-    token.markup = '~'
+    token.markup = TILDE_CHAR
 
     state.pos = state.posMax + 1
     state.posMax = maximum

funnel/utils/markdown/mdit_plugins/sup_tag.py

@@ -17,22 +17,22 @@
 
 __all__ = ['sup_plugin']
 
-CARET_CHAR = 0x5E  # ASCII value for `^`
+CARET_CHAR = '^'
 
 WHITESPACE_RE = re.compile(r'(^|[^\\])(\\\\)*\s')
 UNESCAPE_RE = re.compile(r'\\([ \\!"#$%&\'()*+,.\/:;<=>?@[\]^_`{|}~-])')
 
 
 def tokenize(state: StateInline, silent: bool) -> bool:
     start = state.pos
-    marker = state.srcCharCode[start]
+    ch = state.src[start]
     maximum = state.posMax
     found = False
 
     if silent:
         return False
 
-    if marker != CARET_CHAR:
+    if ch != CARET_CHAR:
         return False
 
     # Don't run any pairs in validation mode
@@ -42,7 +42,7 @@ def tokenize(state: StateInline, silent: bool) -> bool:
     state.pos = start + 1
 
     while state.pos < maximum:
-        if state.srcCharCode[state.pos] == CARET_CHAR:
+        if state.src[state.pos] == CARET_CHAR:
             found = True
             break
         state.md.inline.skipToken(state)
@@ -63,13 +63,13 @@ def tokenize(state: StateInline, silent: bool) -> bool:
 
     # Earlier we checked "not silent", but this implementation does not need it
     token = state.push('sup_open', 'sup', 1)
-    token.markup = '^'
+    token.markup = CARET_CHAR
 
     token = state.push('text', '', 0)
     token.content = UNESCAPE_RE.sub('$1', content)
 
     token = state.push('sup_close', 'sup', -1)
-    token.markup = '^'
+    token.markup = CARET_CHAR
 
     state.pos = state.posMax + 1
     state.posMax = maximum

funnel/utils/markdown/mdit_plugins/toc.py

@@ -20,14 +20,15 @@
 
 from markdown_it import MarkdownIt
 from markdown_it.renderer import OptionsDict, RendererHTML
+from markdown_it.rules_core import StateCore
 from markdown_it.rules_inline import StateInline
 from markdown_it.token import Token
 
 from coaster.utils import make_name
 
 __all__ = ['toc_plugin']
 
-SQUARE_BRACKET_OPEN_CHAR = 0x5B  # ASCII value for `[`
+SQUARE_BRACKET_OPEN_CHAR = '['
 
 defaults: Dict = {
     'include_level': [1, 2, 3, 4, 5, 6],
@@ -197,7 +198,7 @@ def toc_plugin(md: MarkdownIt, **opts) -> None:
 
     def toc(state: StateInline, silent: bool) -> bool:
         # Reject if the token does not start with [
-        if state.srcCharCode[state.pos] != SQUARE_BRACKET_OPEN_CHAR:
+        if state.src[state.pos] != SQUARE_BRACKET_OPEN_CHAR:
             return False
         if silent:
             return False
@@ -253,7 +254,7 @@ def toc_body(
         html = toc_item_to_html(toc, opts, md)
         return html
 
-    def grab_state(state: StateInline):
+    def grab_state(state: StateCore):
         state.env['gstate'] = state
 
     md.core.ruler.push('grab_state', grab_state)

requirements/base.in

@@ -35,7 +35,7 @@ idna
 ijson
 itsdangerous
 linkify-it-py
-markdown-it-py<3.0  # Breaks our plugins, needs refactoring
+markdown-it-py
 mdit-py-plugins
 oauth2
 oauth2client

requirements/base.txt

@@ -1,4 +1,4 @@
-# SHA1:5413ec7fabb7788a7e93e2cc0ed7b97ce62aa936
+# SHA1:36d02851efc3c63077a6ffeb9d21fa8db37894d1
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -253,7 +253,7 @@ markdown==3.4.4
     #   coaster
     #   flask-flatpages
     #   pymdown-extensions
-markdown-it-py==2.2.0
+markdown-it-py==3.0.0
     # via
     #   -r requirements/base.in
     #   mdit-py-plugins

tests/unit/utils/markdown/data/abbr.toml

@@ -1402,7 +1402,7 @@ document = """<h2 id="h:testing-abbreviations-plugin"><a href="#h:testing-abbrev
 ;<abbr title="Abbreviation">ABBR</abbr>;<br />
 ?<abbr title="Abbreviation">ABBR</abbr>?<br />
 @<abbr title="Abbreviation">ABBR</abbr>@<br />
-\\<abbr title="Abbreviation">ABBR</abbr><br />
+\\ABBR<br />
 [<abbr title="Abbreviation">ABBR</abbr>[<br />
 ]<abbr title="Abbreviation">ABBR</abbr>]<br />
 <em><abbr title="Abbreviation">ABBR</abbr></em><br />
@@ -2075,7 +2075,7 @@ abbr = """<h2>Testing abbreviations plugin</h2>
 ;<abbr title="Abbreviation">ABBR</abbr>;
 ?<abbr title="Abbreviation">ABBR</abbr>?
 @<abbr title="Abbreviation">ABBR</abbr>@
-\\<abbr title="Abbreviation">ABBR</abbr><br>
+\\ABBR<br>
 [<abbr title="Abbreviation">ABBR</abbr>[
 ]<abbr title="Abbreviation">ABBR</abbr>]
 <em><abbr title="Abbreviation">ABBR</abbr></em>