Markdown parser phase 2

Makefile

@@ -39,4 +39,4 @@ deps-test:
 tests-data: tests-data-md
 
 tests-data-md:
-	pytest -m update_markdown_data
+	pytest -v -m update_markdown_data

funnel/models/comment.py

@@ -16,7 +16,7 @@
 from . import (
     BaseMixin,
     Mapped,
-    MarkdownColumn,
+    MarkdownColumnNative,
     TSVectorType,
     UuidMixin,
     db,
@@ -220,7 +220,9 @@ class Comment(UuidMixin, BaseMixin, db.Model):
         'Comment', backref=sa.orm.backref('in_reply_to', remote_side='Comment.id')
     )
 
-    _message = MarkdownColumn('message', nullable=False)
+    _message = MarkdownColumnNative(  # type: ignore[has-type]
+        'message', profile='basic', nullable=False
+    )
 
     _state = sa.Column(
         'state',

funnel/models/helpers.py

@@ -13,6 +13,8 @@
 from sqlalchemy.dialects.postgresql.base import (
     RESERVED_WORDS as POSTGRESQL_RESERVED_WORDS,
 )
+from sqlalchemy.ext.mutable import MutableComposite
+from sqlalchemy.orm import composite
 
 from flask import Markup
 from flask import escape as html_escape
@@ -30,6 +32,7 @@
 
 from .. import app
 from ..typing import T
+from ..utils import markdown
 from . import UrlType, db, sa
 
 __all__ = [
@@ -47,6 +50,7 @@
     'quote_autocomplete_like',
     'ImgeeFurl',
     'ImgeeType',
+    'MarkdownColumnNative',
 ]
 
 RESERVED_NAMES: Set[str] = {
@@ -588,3 +592,127 @@ def process_bind_param(self, value, dialect):
             if allowed_schemes and parsed.scheme not in allowed_schemes:
                 raise ValueError("Invalid scheme for the URL")
         return value
+
+
+class MarkdownCompositeNative(MutableComposite):
+    """Represents Markdown text and rendered HTML as a composite column."""
+
+    profile: str
+
+    def __init__(self, text, html=None):
+        """Create a composite."""
+        if html is None:
+            self.text = text  # This will regenerate HTML
+        else:
+            self._text = text
+            self._html = html
+
+    # Return column values for SQLAlchemy to insert into the database
+    def __composite_values__(self):
+        """Return composite values."""
+        return (self._text, self._html)
+
+    # Return a string representation of the text (see class decorator)
+    def __str__(self):
+        """Return string representation."""
+        return self.text or ''
+
+    # Return a HTML representation of the text
+    def __html__(self):
+        """Return HTML representation."""
+        return self._html or ''
+
+    # Return a Markup string of the HTML
+    @property
+    def html(self):
+        """Return HTML as a property."""
+        return Markup(self._html) if self._html is not None else None
+
+    @property
+    def text(self):
+        """Return text as a property."""
+        return self._text
+
+    @text.setter
+    def text(self, value):
+        """Set the text value."""
+        self._text = None if value is None else str(value)
+        self._html = markdown(self._text, self.profile)
+        self.changed()
+
+    def __json__(self) -> Dict[str, Optional[str]]:
+        """Return JSON-compatible rendering of composite."""
+        return {'text': self._text, 'html': self._html}
+
+    # Compare text value
+    def __eq__(self, other):
+        """Compare for equality."""
+        return isinstance(other, MarkdownCompositeNative) and (
+            self.__composite_values__() == other.__composite_values__()
+        )
+
+    def __ne__(self, other):
+        """Compare for inequality."""
+        return not self.__eq__(other)
+
+    # Pickle support methods implemented as per SQLAlchemy documentation, but not
+    # tested here as we don't use them.
+    # https://docs.sqlalchemy.org/en/13/orm/extensions/mutable.html#id1
+
+    def __getstate__(self):
+        """Get state for pickling."""
+        # Return state for pickling
+        return (self._text, self._html)
+
+    def __setstate__(self, state):
+        """Set state from pickle."""
+        # Set state from pickle
+        self._text, self._html = state
+        self.changed()
+
+    def __bool__(self):
+        """Return boolean value."""
+        return bool(self._text)
+
+    @classmethod
+    def coerce(cls, key, value):
+        """Allow a composite column to be assigned a string value."""
+        return cls(value)
+
+
+def markdown_column_native(
+    name: str,
+    deferred: bool = False,
+    group: Optional[str] = None,
+    profile: str = 'basic',
+    **kwargs,
+) -> composite:
+    """
+    Create a composite column that autogenerates HTML from Markdown text.
+
+    Creates two db columns named with ``_html`` and ``_text`` suffixes.
+
+    :param str name: Column name base
+    :param str profile: Config profile for the Markdown processor
+    :param bool deferred: Whether the columns should be deferred by default
+    :param str group: Defer column group
+    :param kwargs: Additional column options, passed to SQLAlchemy's column constructor
+    """
+    # Construct a custom subclass of MarkdownComposite and set the markdown processor
+    # and processor options on it. We'll pass this class to SQLAlchemy's composite
+    # constructor.
+    class CustomMarkdownComposite(MarkdownCompositeNative):
+        pass
+
+    CustomMarkdownComposite.profile = profile
+
+    return composite(
+        CustomMarkdownComposite,
+        sa.Column(name + '_text', sa.UnicodeText, **kwargs),
+        sa.Column(name + '_html', sa.UnicodeText, **kwargs),
+        deferred=deferred,
+        group=group or name,
+    )
+
+
+MarkdownColumnNative = markdown_column_native

funnel/models/update.py

@@ -20,7 +20,7 @@
     BaseScopedIdNameMixin,
     Commentset,
     Mapped,
-    MarkdownColumn,
+    MarkdownColumnNative,
     Project,
     TimestampMixin,
     TSVectorType,
@@ -101,7 +101,9 @@ class Update(UuidMixin, BaseScopedIdNameMixin, TimestampMixin, db.Model):
     )
     parent = sa.orm.synonym('project')
 
-    body = MarkdownColumn('body', nullable=False)
+    body = MarkdownColumnNative(  # type: ignore[has-type]
+        'body', profile='basic', nullable=False
+    )
 
     #: Update number, for Project updates, assigned when the update is published
     number = with_roles(

funnel/models/venue.py

@@ -6,7 +6,14 @@
 
 from coaster.sqlalchemy import add_primary_relationship, with_roles
 
-from . import BaseScopedNameMixin, CoordinatesMixin, MarkdownColumn, UuidMixin, db, sa
+from . import (
+    BaseScopedNameMixin,
+    CoordinatesMixin,
+    MarkdownColumnNative,
+    UuidMixin,
+    db,
+    sa,
+)
 from .helpers import reopen
 from .project import Project
 from .project_membership import project_child_role_map
@@ -25,7 +32,9 @@ class Venue(UuidMixin, BaseScopedNameMixin, CoordinatesMixin, db.Model):
         grants_via={None: project_child_role_map},
     )
     parent = sa.orm.synonym('project')
-    description = MarkdownColumn('description', default='', nullable=False)
+    description = MarkdownColumnNative(  # type: ignore[has-type]
+        'description', profile='basic', default='', nullable=False
+    )
     address1 = sa.Column(sa.Unicode(160), default='', nullable=False)
     address2 = sa.Column(sa.Unicode(160), default='', nullable=False)
     city = sa.Column(sa.Unicode(30), default='', nullable=False)
@@ -104,7 +113,9 @@ class VenueRoom(UuidMixin, BaseScopedNameMixin, db.Model):
         grants_via={None: set(project_child_role_map.values())},
     )
     parent = sa.orm.synonym('venue')
-    description = MarkdownColumn('description', default='', nullable=False)
+    description = MarkdownColumnNative(  # type: ignore[has-type]
+        'description', profile='basic', default='', nullable=False
+    )
     bgcolor = sa.Column(sa.Unicode(6), nullable=False, default='229922')
 
     seq = sa.Column(sa.Integer, nullable=False)

funnel/utils/markdown/base.py

@@ -1,14 +1,15 @@
 """Base files for markdown parser."""
 # pylint: disable=too-many-arguments
 
-from typing import Dict, List, Optional, Union, overload
+from typing import Any, Dict, List, Optional, Union, overload
+import json
 
 from markdown_it import MarkdownIt
 from markupsafe import Markup
 
 from coaster.utils.text import normalize_spaces_multiline
 
-from .extmap import markdown_extensions
+from .profiles import plugin_configs, plugins, profiles
 
 __all__ = ['markdown']
 
@@ -32,63 +33,88 @@
 
 
 @overload
-def markdown(
-    text: None,
-    extensions: Union[List[str], None] = None,
-    extension_configs: Optional[Dict[str, str]] = None,
-    # TODO: Extend to accept helpers.EXT_CONFIG_TYPE (Dict)
-) -> None:
+def markdown(text: None, profile: Union[str, Dict[str, Any]]) -> None:
     ...
 
 
 @overload
-def markdown(
-    text: str,
-    extensions: Union[List[str], None] = None,
-    extension_configs: Optional[Dict[str, str]] = None,
-    # TODO: Extend to accept helpers.EXT_CONFIG_TYPE (Dict)
-) -> Markup:
+def markdown(text: str, profile: Union[str, Dict[str, Any]]) -> Markup:
     ...
 
 
 def markdown(
-    text: Optional[str],
-    extensions: Union[List[str], None] = None,
-    extension_configs: Optional[Dict[str, str]] = None,
-    # TODO: Extend to accept helpers.EXT_CONFIG_TYPE (Dict)
+    text: Optional[str], profile: Union[str, Dict[str, Any]]
 ) -> Optional[Markup]:
     """
     Markdown parser compliant with Commonmark+GFM using markdown-it-py.
 
-    :param bool linkify: Whether to convert naked URLs into links
-    :param list extensions: List of Markdown extensions to be enabled
-    :param dict extension_configs: Config for Markdown extensions
+    :param str|dict profile: Config profile to use
     """
     if text is None:
         return None
 
     # Replace invisible characters with spaces
     text = normalize_spaces_multiline(text)
 
-    md = MarkdownIt(
-        'gfm-like',
-        {
-            'breaks': True,
-            'linkify': True,
-            'typographer': True,
-        },
-    ).enable(['smartquotes'])
-
-    md.linkify.set({'fuzzy_link': False, 'fuzzy_email': False})
-
-    if extensions is None:
-        extensions = default_markdown_extensions
-
-    for e in extensions:
-        if e in markdown_extensions:
-            ext_config = markdown_extensions[e].default_config
-            if extension_configs is not None and e in extension_configs:
-                ext_config = markdown_extensions[e].config(extension_configs[e])
-            md.use(markdown_extensions[e].ext, **ext_config)
-
-    return Markup(md.render(text))  # type: ignore[arg-type]
+    if isinstance(profile, str):
+        try:
+            _profile = profiles[profile]
+        except KeyError as exc:
+            raise KeyError(
+                f'Wrong markdown config profile "{profile}". Check name.'
+            ) from exc
+    elif isinstance(profile, dict):
+        _profile = profile
+    else:
+        raise TypeError('Wrong type - profile has to be either str or dict')
+
+    args = _profile.get('args', ())
+
+    md = MarkdownIt(*args)
+
+    funnel_config = _profile.get('funnel_config', {})
+
+    if md.linkify is not None:
+        md.linkify.set({'fuzzy_link': False, 'fuzzy_email': False})
+
+    for action in ['enableOnly', 'enable', 'disable']:
+        if action in funnel_config:
+            getattr(md, action)(funnel_config[action])
+
+    for e in _profile.get('plugins', []):
+        try:
+            ext = plugins[e]
+        except KeyError as exc:
+            raise KeyError(
+                f'Wrong markdown-it-py plugin key "{e}". Check name.'
+            ) from exc
+        md.use(ext, **plugin_configs.get(e, {}))
+
+    # type: ignore[arg-type]
+    return Markup(getattr(md, funnel_config.get('render_with', 'render'))(text))
+
+
+def _print_rules(md: MarkdownIt, active: str = None):
+    """Debug function to be removed before merge."""
+    rules = {'all_rules': md.get_all_rules(), 'active_rules': {}}
+    for p, pr in profiles.items():
+        m = MarkdownIt(*pr.get('args', ()))
+        fc = pr.get('funnel_config', {})
+        if m.linkify is not None:
+            m.linkify.set({'fuzzy_link': False, 'fuzzy_email': False})
+        for action in ['enableOnly', 'enable', 'disable']:
+            if action in fc:
+                getattr(m, action)(fc[action])
+        for e in pr.get('plugins', []):
+            try:
+                ext = plugins[e]
+            except KeyError as exc:
+                raise KeyError(
+                    f'Wrong markdown-it-py plugin key "{e}". Check name.'
+                ) from exc
+            m.use(ext, **plugin_configs.get(e, {}))
+        rules['active_rules'][p] = m.get_active_rules()
+    if active is not None:
+        print(json.dumps(rules['active_rules'][active], indent=2))  # noqa: T201
+    else:
+        print(json.dumps(rules, indent=2))  # noqa: T201