From 6a143153d2402b25406e7725204824de91055361 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 20 Sep 2022 04:19:50 +0530 Subject: [PATCH 01/72] New markdown config profiles structure --- funnel/utils/markdown/base.py | 73 ++++++++++------------- funnel/utils/markdown/profiles.py | 96 +++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 43 deletions(-) create mode 100644 funnel/utils/markdown/profiles.py diff --git a/funnel/utils/markdown/base.py b/funnel/utils/markdown/base.py index ca030fe52..d49d00531 100644 --- a/funnel/utils/markdown/base.py +++ b/funnel/utils/markdown/base.py @@ -1,14 +1,14 @@ """Base files for markdown parser.""" # pylint: disable=too-many-arguments -from typing import Dict, List, Optional, Union, overload +from typing import List, Optional, overload from markdown_it import MarkdownIt from markupsafe import Markup from coaster.utils.text import normalize_spaces_multiline -from .extmap import markdown_extensions +from .profiles import plugin_configs, plugins, profiles __all__ = ['markdown'] @@ -32,37 +32,20 @@ @overload -def markdown( - text: None, - extensions: Union[List[str], None] = None, - extension_configs: Optional[Dict[str, str]] = None, - # TODO: Extend to accept helpers.EXT_CONFIG_TYPE (Dict) -) -> None: +def markdown(text: None, profile: str = 'proposal') -> None: ... @overload -def markdown( - text: str, - extensions: Union[List[str], None] = None, - extension_configs: Optional[Dict[str, str]] = None, - # TODO: Extend to accept helpers.EXT_CONFIG_TYPE (Dict) -) -> Markup: +def markdown(text: str, profile: str = 'proposal') -> Markup: ... -def markdown( - text: Optional[str], - extensions: Union[List[str], None] = None, - extension_configs: Optional[Dict[str, str]] = None, - # TODO: Extend to accept helpers.EXT_CONFIG_TYPE (Dict) -) -> Optional[Markup]: +def markdown(text: Optional[str], profile: str = 'proposal') -> Optional[Markup]: """ Markdown parser compliant with Commonmark+GFM using markdown-it-py. - :param bool linkify: Whether to convert naked URLs into links - :param list extensions: List of Markdown extensions to be enabled - :param dict extension_configs: Config for Markdown extensions + :param bool profile: Config profile to use """ if text is None: return None @@ -70,25 +53,29 @@ def markdown( # Replace invisible characters with spaces text = normalize_spaces_multiline(text) - md = MarkdownIt( - 'gfm-like', - { - 'breaks': True, - 'linkify': True, - 'typographer': True, - }, - ).enable(['smartquotes']) - - md.linkify.set({'fuzzy_link': False, 'fuzzy_email': False}) - - if extensions is None: - extensions = default_markdown_extensions - - for e in extensions: - if e in markdown_extensions: - ext_config = markdown_extensions[e].default_config - if extension_configs is not None and e in extension_configs: - ext_config = markdown_extensions[e].config(extension_configs[e]) - md.use(markdown_extensions[e].ext, **ext_config) + if profile not in profiles: + raise KeyError(f'Wrong markdown config profile "{profile}". Check name.') + + args = profiles[profile].get('args', ()) + + md = MarkdownIt(*args) + + funnel_config = profiles[profile].get('funnel_config', {}) + + if md.linkify is not None: + md.linkify.set({'fuzzy_link': False, 'fuzzy_email': False}) + + for action in ['enable', 'disable']: + if action in funnel_config: + md.enable(funnel_config[action]) + + for e in profiles[profile].get('plugins', []): + try: + ext = plugins[e] + except KeyError as exc: + raise KeyError( + f'Wrong markdown-it-py plugin key "{e}". Check name.' + ) from exc + md.use(ext, **plugin_configs.get(e, {})) return Markup(md.render(text)) # type: ignore[arg-type] diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py new file mode 100644 index 000000000..c97aed90d --- /dev/null +++ b/funnel/utils/markdown/profiles.py @@ -0,0 +1,96 @@ +"""Config profiles for markdown parser.""" + +from typing import Any, Callable, Dict + +from mdit_py_plugins import anchors, footnote, tasklists + +from coaster.utils import make_name + +__all__ = ['profiles', 'plugins', 'plugin_configs'] + +plugins: Dict[str, Callable] = { + 'footnote': footnote.footnote_plugin, + 'heading_anchors': anchors.anchors_plugin, + 'tasklists': tasklists.tasklists_plugin, +} + +plugin_configs: Dict[str, Dict[str, Any]] = { + 'heading_anchors': { + 'min_level': 1, + 'max_level': 3, + 'slug_func': lambda x, **options: 'h:' + make_name(x, **options), + 'permalink': True, + }, + 'tasklists': {'enabled': True, 'label': True, 'label_after': False}, +} + +default_funnel_options = { + 'html': False, + 'linkify': True, + 'typographer': True, + 'breaks': True, +} +default_funnel_args = ('gfm-like', default_funnel_options) +default_funnel_config = {'enable': ['smartquotes']} +default_plugins = [ + 'footnote', + 'heading_anchors', + 'tasklists', +] + +profiles: Dict[str, Dict[str, Any]] = { + """ + Config profiles. + + Format: { + 'args': ( + config: str | preset.make(), + options_update: Mapping | None + ), + 'funnel_config' : { # Optional + 'enable': [], + 'disable': [], + 'renderInline': False + } + } + """ + 'comment': { + 'args': ('commonmark', default_funnel_options), + 'funnel_config': default_funnel_config, + 'plugins': ['footnote'], + }, + 'profile': { + 'args': default_funnel_args, + 'funnel_config': default_funnel_config, + 'plugins': default_plugins, + }, + 'project': { + 'args': default_funnel_args, + 'funnel_config': default_funnel_config, + 'plugins': default_plugins, + }, + 'proposal': { + 'args': default_funnel_args, + 'funnel_config': default_funnel_config, + 'plugins': default_plugins, + }, + 'session': { + 'args': default_funnel_args, + 'funnel_config': default_funnel_config, + 'plugins': default_plugins, + }, + 'update': { + 'args': ('commonmark', default_funnel_options), + 'funnel_config': default_funnel_config, + 'plugins': default_plugins, + }, + 'venue': { + 'args': ('commonmark', default_funnel_options), + 'funnel_config': default_funnel_config, + 'plugins': default_plugins, + }, + 'single-line': { + 'args': ('zero', {}), + 'funnel_config': {'enable': ['emphasize', 'backticks'], 'renderInline': True}, + }, +} From 7c2215bde3bfaf9d1043b4fb2f0b2a5b55633ea2 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 20 Sep 2022 12:11:42 +0530 Subject: [PATCH 02/72] Remove smartquotes. Correct profiles for update and venue --- funnel/utils/markdown/profiles.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index c97aed90d..6da3792a7 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -31,7 +31,7 @@ 'breaks': True, } default_funnel_args = ('gfm-like', default_funnel_options) -default_funnel_config = {'enable': ['smartquotes']} +default_funnel_config: Dict = {} default_plugins = [ 'footnote', 'heading_anchors', @@ -82,12 +82,12 @@ 'update': { 'args': ('commonmark', default_funnel_options), 'funnel_config': default_funnel_config, - 'plugins': default_plugins, + 'plugins': [], }, 'venue': { 'args': ('commonmark', default_funnel_options), 'funnel_config': default_funnel_config, - 'plugins': default_plugins, + 'plugins': [], }, 'single-line': { 'args': ('zero', {}), From a458903e9dbcc1e0a8c6e97d71ae33ea1f0f3cc3 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 20 Sep 2022 22:52:16 +0530 Subject: [PATCH 03/72] Generic markdown profiles: basic, document, text-field. Future provision for preset configs in markdown-it-py style class.make(). --- funnel/utils/markdown/base.py | 40 +++++++++++--- funnel/utils/markdown/presets.py | 13 +++++ funnel/utils/markdown/profiles.py | 89 +++++++++++-------------------- funnel/views/api/markdown.py | 2 +- 4 files changed, 80 insertions(+), 64 deletions(-) create mode 100644 funnel/utils/markdown/presets.py diff --git a/funnel/utils/markdown/base.py b/funnel/utils/markdown/base.py index d49d00531..053573789 100644 --- a/funnel/utils/markdown/base.py +++ b/funnel/utils/markdown/base.py @@ -2,6 +2,7 @@ # pylint: disable=too-many-arguments from typing import List, Optional, overload +import json from markdown_it import MarkdownIt from markupsafe import Markup @@ -32,16 +33,16 @@ @overload -def markdown(text: None, profile: str = 'proposal') -> None: +def markdown(text: None, profile: str) -> None: ... @overload -def markdown(text: str, profile: str = 'proposal') -> Markup: +def markdown(text: str, profile: str) -> Markup: ... -def markdown(text: Optional[str], profile: str = 'proposal') -> Optional[Markup]: +def markdown(text: Optional[str], profile: str) -> Optional[Markup]: """ Markdown parser compliant with Commonmark+GFM using markdown-it-py. @@ -65,9 +66,9 @@ def markdown(text: Optional[str], profile: str = 'proposal') -> Optional[Markup] if md.linkify is not None: md.linkify.set({'fuzzy_link': False, 'fuzzy_email': False}) - for action in ['enable', 'disable']: + for action in ['enableOnly', 'enable', 'disable']: if action in funnel_config: - md.enable(funnel_config[action]) + getattr(md, action)(funnel_config[action]) for e in profiles[profile].get('plugins', []): try: @@ -78,4 +79,31 @@ def markdown(text: Optional[str], profile: str = 'proposal') -> Optional[Markup] ) from exc md.use(ext, **plugin_configs.get(e, {})) - return Markup(md.render(text)) # type: ignore[arg-type] + # type: ignore[arg-type] + return Markup(getattr(md, funnel_config.get('render_with', 'render'))(text)) + + +def _print_rules(md: MarkdownIt, active: str = None): + """Debug function to be removed before merge.""" + rules = {'all_rules': md.get_all_rules(), 'active_rules': {}} + for p, pr in profiles.items(): + m = MarkdownIt(*pr.get('args', ())) + fc = pr.get('funnel_config', {}) + if m.linkify is not None: + m.linkify.set({'fuzzy_link': False, 'fuzzy_email': False}) + for action in ['enableOnly', 'enable', 'disable']: + if action in fc: + getattr(m, action)(fc[action]) + for e in pr.get('plugins', []): + try: + ext = plugins[e] + except KeyError as exc: + raise KeyError( + f'Wrong markdown-it-py plugin key "{e}". Check name.' + ) from exc + m.use(ext, **plugin_configs.get(e, {})) + rules['active_rules'][p] = m.get_active_rules() + if active is not None: + print(json.dumps(rules['active_rules'][active], indent=2)) # noqa: T201 + else: + print(json.dumps(rules, indent=2)) # noqa: T201 diff --git a/funnel/utils/markdown/presets.py b/funnel/utils/markdown/presets.py new file mode 100644 index 000000000..d7d74be07 --- /dev/null +++ b/funnel/utils/markdown/presets.py @@ -0,0 +1,13 @@ +"""markdown-it-py style presets for funnel.""" + +from markdown_it.presets import gfm_like + +__all__ = [] # type: ignore[var-annotated] + + +class custom_profile: # noqa: N801 + @staticmethod + def make(): + config = gfm_like.make() + # ... options to process single line + return config diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 6da3792a7..ee4e77611 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -30,67 +30,42 @@ 'typographer': True, 'breaks': True, } -default_funnel_args = ('gfm-like', default_funnel_options) -default_funnel_config: Dict = {} -default_plugins = [ - 'footnote', - 'heading_anchors', - 'tasklists', -] -profiles: Dict[str, Dict[str, Any]] = { - """ - Config profiles. - Format: { - 'args': ( - config: str | preset.make(), - options_update: Mapping | None - ), - 'funnel_config' : { # Optional - 'enable': [], - 'disable': [], - 'renderInline': False - } - } - """ - 'comment': { - 'args': ('commonmark', default_funnel_options), - 'funnel_config': default_funnel_config, - 'plugins': ['footnote'], - }, - 'profile': { - 'args': default_funnel_args, - 'funnel_config': default_funnel_config, - 'plugins': default_plugins, - }, - 'project': { - 'args': default_funnel_args, - 'funnel_config': default_funnel_config, - 'plugins': default_plugins, - }, - 'proposal': { - 'args': default_funnel_args, - 'funnel_config': default_funnel_config, - 'plugins': default_plugins, - }, - 'session': { - 'args': default_funnel_args, - 'funnel_config': default_funnel_config, - 'plugins': default_plugins, - }, - 'update': { - 'args': ('commonmark', default_funnel_options), - 'funnel_config': default_funnel_config, +# Config profiles. +# +# Format: { +# 'args': ( +# config:str | preset.make(), +# options_update: Mapping | None +# ), +# 'funnel_config' : { # Optional +# 'enable': List = [], +# 'disable': List = [], +# 'enableOnly': List = [], +# 'render_with': str = 'render' +# } +# } + +profiles: Dict[str, Dict[str, Any]] = { + 'basic': { + 'args': ('gfm-like', default_funnel_options), 'plugins': [], + 'funnel_config': {'disable': ['table']}, }, - 'venue': { - 'args': ('commonmark', default_funnel_options), - 'funnel_config': default_funnel_config, - 'plugins': [], + 'document': { + 'args': ('gfm-like', default_funnel_options), + 'plugins': [ + 'footnote', + 'heading_anchors', + 'tasklists', + ], }, - 'single-line': { - 'args': ('zero', {}), - 'funnel_config': {'enable': ['emphasize', 'backticks'], 'renderInline': True}, + 'text-field': { + 'args': ('zero', {'breaks': False}), + 'funnel_config': { + 'enable': ['emphasis', 'backticks'], + 'render_with': 'renderInline', + }, }, } diff --git a/funnel/views/api/markdown.py b/funnel/views/api/markdown.py index bacb4d93f..e4102d44f 100644 --- a/funnel/views/api/markdown.py +++ b/funnel/views/api/markdown.py @@ -17,7 +17,7 @@ def markdown_preview() -> ReturnView: # mtype = request.form.get('type') text = request.form.get('text') - html = markdown(text) + html = markdown(text, 'document') return { 'status': 'ok', From 6cc62a0ed223d7a9fd12a10d96c671c42c3abde0 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 22 Sep 2022 16:24:01 +0530 Subject: [PATCH 04/72] Introduced MarkdownColumnNative, that uses the new markdown config profiles. WIP #1485 --- funnel/models/comment.py | 4 +- funnel/models/helpers.py | 128 +++++++++++++++++++++++++++++++++++++++ funnel/models/update.py | 6 +- 3 files changed, 134 insertions(+), 4 deletions(-) diff --git a/funnel/models/comment.py b/funnel/models/comment.py index b22ad50e7..d2cb6afbc 100644 --- a/funnel/models/comment.py +++ b/funnel/models/comment.py @@ -16,7 +16,7 @@ from . import ( BaseMixin, Mapped, - MarkdownColumn, + MarkdownColumnNative, TSVectorType, UuidMixin, db, @@ -220,7 +220,7 @@ class Comment(UuidMixin, BaseMixin, db.Model): 'Comment', backref=sa.orm.backref('in_reply_to', remote_side='Comment.id') ) - _message = MarkdownColumn('message', nullable=False) + _message = MarkdownColumnNative('message', profile='basic', nullable=False) _state = sa.Column( 'state', diff --git a/funnel/models/helpers.py b/funnel/models/helpers.py index 2883ed0cc..1ac6d0502 100644 --- a/funnel/models/helpers.py +++ b/funnel/models/helpers.py @@ -13,6 +13,8 @@ from sqlalchemy.dialects.postgresql.base import ( RESERVED_WORDS as POSTGRESQL_RESERVED_WORDS, ) +from sqlalchemy.ext.mutable import MutableComposite +from sqlalchemy.orm import composite from flask import Markup from flask import escape as html_escape @@ -30,6 +32,7 @@ from .. import app from ..typing import T +from ..utils import markdown from . import UrlType, db, sa __all__ = [ @@ -47,6 +50,7 @@ 'quote_autocomplete_like', 'ImgeeFurl', 'ImgeeType', + 'MarkdownColumnNative', ] RESERVED_NAMES: Set[str] = { @@ -588,3 +592,127 @@ def process_bind_param(self, value, dialect): if allowed_schemes and parsed.scheme not in allowed_schemes: raise ValueError("Invalid scheme for the URL") return value + + +class MarkdownCompositeNative(MutableComposite): + """Represents Markdown text and rendered HTML as a composite column.""" + + profile: str + + def __init__(self, text, html=None): + """Create a composite.""" + if html is None: + self.text = text # This will regenerate HTML + else: + self._text = text + self._html = html + + # Return column values for SQLAlchemy to insert into the database + def __composite_values__(self): + """Return composite values.""" + return (self._text, self._html) + + # Return a string representation of the text (see class decorator) + def __str__(self): + """Return string representation.""" + return self.text or '' + + # Return a HTML representation of the text + def __html__(self): + """Return HTML representation.""" + return self._html or '' + + # Return a Markup string of the HTML + @property + def html(self): + """Return HTML as a property.""" + return Markup(self._html) if self._html is not None else None + + @property + def text(self): + """Return text as a property.""" + return self._text + + @text.setter + def text(self, value): + """Set the text value.""" + self._text = None if value is None else str(value) + self._html = markdown(self._text, self.profile) + self.changed() + + def __json__(self) -> Dict[str, Optional[str]]: + """Return JSON-compatible rendering of composite.""" + return {'text': self._text, 'html': self._html} + + # Compare text value + def __eq__(self, other): + """Compare for equality.""" + return isinstance(other, MarkdownCompositeNative) and ( + self.__composite_values__() == other.__composite_values__() + ) + + def __ne__(self, other): + """Compare for inequality.""" + return not self.__eq__(other) + + # Pickle support methods implemented as per SQLAlchemy documentation, but not + # tested here as we don't use them. + # https://docs.sqlalchemy.org/en/13/orm/extensions/mutable.html#id1 + + def __getstate__(self): + """Get state for pickling.""" + # Return state for pickling + return (self._text, self._html) + + def __setstate__(self, state): + """Set state from pickle.""" + # Set state from pickle + self._text, self._html = state + self.changed() + + def __bool__(self): + """Return boolean value.""" + return bool(self._text) + + @classmethod + def coerce(cls, key, value): + """Allow a composite column to be assigned a string value.""" + return cls(value) + + +def markdown_column_native( + name: str, + deferred: bool = False, + group: Optional[str] = None, + profile: str = 'basic', + **kwargs, +) -> composite: + """ + Create a composite column that autogenerates HTML from Markdown text. + + Creates two db columns named with ``_html`` and ``_text`` suffixes. + + :param str name: Column name base + :param str profile: Config profile for the Markdown processor + :param bool deferred: Whether the columns should be deferred by default + :param str group: Defer column group + :param kwargs: Additional column options, passed to SQLAlchemy's column constructor + """ + # Construct a custom subclass of MarkdownComposite and set the markdown processor + # and processor options on it. We'll pass this class to SQLAlchemy's composite + # constructor. + class CustomMarkdownComposite(MarkdownCompositeNative): + pass + + CustomMarkdownComposite.profile = profile + + return composite( + CustomMarkdownComposite, + sa.Column(name + '_text', sa.UnicodeText, **kwargs), + sa.Column(name + '_html', sa.UnicodeText, **kwargs), + deferred=deferred, + group=group or name, + ) + + +MarkdownColumnNative = markdown_column_native diff --git a/funnel/models/update.py b/funnel/models/update.py index d9c7dc0fb..95a234b17 100644 --- a/funnel/models/update.py +++ b/funnel/models/update.py @@ -20,7 +20,7 @@ BaseScopedIdNameMixin, Commentset, Mapped, - MarkdownColumn, + MarkdownColumnNative, Project, TimestampMixin, TSVectorType, @@ -101,7 +101,9 @@ class Update(UuidMixin, BaseScopedIdNameMixin, TimestampMixin, db.Model): ) parent = sa.orm.synonym('project') - body = MarkdownColumn('body', nullable=False) + body = MarkdownColumnNative( # type: ignore[has-type] + 'body', profile='basic', nullable=False + ) #: Update number, for Project updates, assigned when the update is published number = with_roles( From d874fd79c441d824cde681d38fa45901137b4376 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 22 Sep 2022 16:55:51 +0530 Subject: [PATCH 05/72] Use MarkdownColumnNative for venue(basic) and venue_room(document, for some experimental testing on production, since this is a dormant field) --- funnel/models/comment.py | 4 +++- funnel/models/venue.py | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/funnel/models/comment.py b/funnel/models/comment.py index d2cb6afbc..f38dacca4 100644 --- a/funnel/models/comment.py +++ b/funnel/models/comment.py @@ -220,7 +220,9 @@ class Comment(UuidMixin, BaseMixin, db.Model): 'Comment', backref=sa.orm.backref('in_reply_to', remote_side='Comment.id') ) - _message = MarkdownColumnNative('message', profile='basic', nullable=False) + _message = MarkdownColumnNative( # type: ignore[has-type] + 'message', profile='basic', nullable=False + ) _state = sa.Column( 'state', diff --git a/funnel/models/venue.py b/funnel/models/venue.py index 45f33abfa..a50abd58f 100644 --- a/funnel/models/venue.py +++ b/funnel/models/venue.py @@ -6,7 +6,14 @@ from coaster.sqlalchemy import add_primary_relationship, with_roles -from . import BaseScopedNameMixin, CoordinatesMixin, MarkdownColumn, UuidMixin, db, sa +from . import ( + BaseScopedNameMixin, + CoordinatesMixin, + MarkdownColumnNative, + UuidMixin, + db, + sa, +) from .helpers import reopen from .project import Project from .project_membership import project_child_role_map @@ -25,7 +32,9 @@ class Venue(UuidMixin, BaseScopedNameMixin, CoordinatesMixin, db.Model): grants_via={None: project_child_role_map}, ) parent = sa.orm.synonym('project') - description = MarkdownColumn('description', default='', nullable=False) + description = MarkdownColumnNative( + 'description', profile='basic', default='', nullable=False + ) address1 = sa.Column(sa.Unicode(160), default='', nullable=False) address2 = sa.Column(sa.Unicode(160), default='', nullable=False) city = sa.Column(sa.Unicode(30), default='', nullable=False) @@ -104,7 +113,9 @@ class VenueRoom(UuidMixin, BaseScopedNameMixin, db.Model): grants_via={None: set(project_child_role_map.values())}, ) parent = sa.orm.synonym('venue') - description = MarkdownColumn('description', default='', nullable=False) + description = MarkdownColumnNative( # type: ignore[has-type] + 'description', profile='basic', default='', nullable=False + ) bgcolor = sa.Column(sa.Unicode(6), nullable=False, default='229922') seq = sa.Column(sa.Integer, nullable=False) From 0f2852f8ba692082142a0be5ffe5dc394a5d83d9 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 23 Sep 2022 01:26:54 +0530 Subject: [PATCH 06/72] Removed funnel.utils.markdown.extmap #1483 --- funnel/utils/markdown/extmap.py | 37 --------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 funnel/utils/markdown/extmap.py diff --git a/funnel/utils/markdown/extmap.py b/funnel/utils/markdown/extmap.py deleted file mode 100644 index 3c722b834..000000000 --- a/funnel/utils/markdown/extmap.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Extension map for markdown-it-py to be used by markdown parser.""" - -from typing import Dict - -from mdit_py_plugins import anchors, footnote, tasklists - -from coaster.utils import make_name - -from .helpers import MDExtension - -__all__ = ['markdown_extensions'] - -markdown_extensions: Dict[str, MDExtension] = { - 'footnote': MDExtension(footnote.footnote_plugin), - 'heading_anchors': MDExtension(anchors.anchors_plugin), - 'tasklists': MDExtension(tasklists.tasklists_plugin), -} - -markdown_extensions['heading_anchors'].set_config( - None, - { - 'min_level': 1, - 'max_level': 3, - 'slug_func': lambda x, **options: 'h:' + make_name(x, **options), - 'permalink': True, - }, -) - - -markdown_extensions['tasklists'].set_config( - None, {'enabled': False, 'label': False, 'label_after': False} -) - -# TODO: These plugins will be integrated later with more robust edge-case testing -# markdown_extensions['ins'] = MDExtension(mdit_plugins.ins_plugin) # type: ignore -# markdown_extensions['del'] = MDExtension(mdit_plugins.del_sub_plugin) -# markdown_extensions['sup'] = MDExtension(mdit_plugins.sup_plugin) From 08f67d500a71c98625bf487e54369597b57a35b5 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Sat, 24 Sep 2022 02:32:21 +0530 Subject: [PATCH 07/72] Revamp markdown tests for new config profile structure #1486 --- Makefile | 2 +- funnel/models/venue.py | 2 +- funnel/utils/markdown/base.py | 31 ++-- funnel/utils/markdown/profiles.py | 8 +- tests/data/markdown/basic.toml | 6 +- tests/data/markdown/code.toml | 6 +- .../{ext_ins.toml => ext_ins.toml.disabled} | 2 +- tests/data/markdown/footnotes.toml | 21 ++- tests/data/markdown/headings.toml | 19 ++- tests/data/markdown/images.toml | 6 +- tests/data/markdown/links.toml | 6 +- tests/data/markdown/lists.toml | 6 +- tests/data/markdown/tables.toml | 6 +- tests/data/markdown/template.html | 4 +- tests/unit/utils/test_markdown.py | 151 +++++++++--------- 15 files changed, 154 insertions(+), 122 deletions(-) rename tests/data/markdown/{ext_ins.toml => ext_ins.toml.disabled} (99%) diff --git a/Makefile b/Makefile index 59c99b84d..5d2357021 100644 --- a/Makefile +++ b/Makefile @@ -39,4 +39,4 @@ deps-test: tests-data: tests-data-md tests-data-md: - pytest -m update_markdown_data + pytest -v -m update_markdown_data diff --git a/funnel/models/venue.py b/funnel/models/venue.py index a50abd58f..34a89669c 100644 --- a/funnel/models/venue.py +++ b/funnel/models/venue.py @@ -32,7 +32,7 @@ class Venue(UuidMixin, BaseScopedNameMixin, CoordinatesMixin, db.Model): grants_via={None: project_child_role_map}, ) parent = sa.orm.synonym('project') - description = MarkdownColumnNative( + description = MarkdownColumnNative( # type: ignore[has-type] 'description', profile='basic', default='', nullable=False ) address1 = sa.Column(sa.Unicode(160), default='', nullable=False) diff --git a/funnel/utils/markdown/base.py b/funnel/utils/markdown/base.py index 053573789..23aa92f92 100644 --- a/funnel/utils/markdown/base.py +++ b/funnel/utils/markdown/base.py @@ -1,7 +1,7 @@ """Base files for markdown parser.""" # pylint: disable=too-many-arguments -from typing import List, Optional, overload +from typing import Any, Dict, List, Optional, Union, overload import json from markdown_it import MarkdownIt @@ -33,20 +33,22 @@ @overload -def markdown(text: None, profile: str) -> None: +def markdown(text: None, profile: Union[str, Dict[str, Any]]) -> None: ... @overload -def markdown(text: str, profile: str) -> Markup: +def markdown(text: str, profile: Union[str, Dict[str, Any]]) -> Markup: ... -def markdown(text: Optional[str], profile: str) -> Optional[Markup]: +def markdown( + text: Optional[str], profile: Union[str, Dict[str, Any]] +) -> Optional[Markup]: """ Markdown parser compliant with Commonmark+GFM using markdown-it-py. - :param bool profile: Config profile to use + :param str|dict profile: Config profile to use """ if text is None: return None @@ -54,14 +56,23 @@ def markdown(text: Optional[str], profile: str) -> Optional[Markup]: # Replace invisible characters with spaces text = normalize_spaces_multiline(text) - if profile not in profiles: - raise KeyError(f'Wrong markdown config profile "{profile}". Check name.') + if isinstance(profile, str): + try: + _profile = profiles[profile] + except KeyError as exc: + raise KeyError( + f'Wrong markdown config profile "{profile}". Check name.' + ) from exc + elif isinstance(profile, dict): + _profile = profile + else: + raise TypeError('Wrong type - profile has to be either str or dict') - args = profiles[profile].get('args', ()) + args = _profile.get('args', ()) md = MarkdownIt(*args) - funnel_config = profiles[profile].get('funnel_config', {}) + funnel_config = _profile.get('funnel_config', {}) if md.linkify is not None: md.linkify.set({'fuzzy_link': False, 'fuzzy_email': False}) @@ -70,7 +81,7 @@ def markdown(text: Optional[str], profile: str) -> Optional[Markup]: if action in funnel_config: getattr(md, action)(funnel_config[action]) - for e in profiles[profile].get('plugins', []): + for e in _profile.get('plugins', []): try: ext = plugins[e] except KeyError as exc: diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index ee4e77611..7eb7df66d 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -6,7 +6,7 @@ from coaster.utils import make_name -__all__ = ['profiles', 'plugins', 'plugin_configs'] +__all__ = ['profiles', 'plugins', 'plugin_configs', 'default_markdown_options'] plugins: Dict[str, Callable] = { 'footnote': footnote.footnote_plugin, @@ -24,7 +24,7 @@ 'tasklists': {'enabled': True, 'label': True, 'label_after': False}, } -default_funnel_options = { +default_markdown_options = { 'html': False, 'linkify': True, 'typographer': True, @@ -49,12 +49,12 @@ profiles: Dict[str, Dict[str, Any]] = { 'basic': { - 'args': ('gfm-like', default_funnel_options), + 'args': ('gfm-like', default_markdown_options), 'plugins': [], 'funnel_config': {'disable': ['table']}, }, 'document': { - 'args': ('gfm-like', default_funnel_options), + 'args': ('gfm-like', default_markdown_options), 'plugins': [ 'footnote', 'heading_anchors', diff --git a/tests/data/markdown/basic.toml b/tests/data/markdown/basic.toml index 483479c54..88b1fc638 100644 --- a/tests/data/markdown/basic.toml +++ b/tests/data/markdown/basic.toml @@ -76,8 +76,8 @@ ___ """ [config] -configs = [ "", "no_ext",] +profiles = [ "basic", "document",] [expected_output] -"" = "

Basic markup

\n

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

\n

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
\nIt also has a newline break here!!!

\n

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

\n

__Bold without closure does not work
\n**Bold without closure does not work
\n_Emphasis without closure does not work
\n*Emphasis without closure does not work

\n

Bold without closure
\non the same line
\ncarries forward to
\nthis is text that should strike off
\nmultiple consecutive lines

\n

Bold without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Horizontal Rules

\n
\n

Above is a horizontal rule using hyphens.
\nThis is text that should strike off

\n
\n

Above is a horizontal rule using asterisks.
\nBelow is a horizontal rule using underscores.

\n
\n

Links

\n

Here is a link to hasgeek

\n

Link to funnel with the title ‘Hasgeek’

\n

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

\n
\n

Markdown-it typography

\n

The results of the below depend on the typographer options enabled by us in the markdown-it-py parser, if typographer=True has been passed to it.

\n

The below should convert if replacements has been enabled.

\n

(c) (C) (r) (R) (tm) (TM) (p) (P) +-
\ntest.. test... test..... test?..... test!....
\n!!!!!! ???? ,, -- ---

\n

The below should convert the quotes if smartquotes has been enabled.

\n

“Smartypants, double quotes” and ‘single quotes’

\n
\n

Blockquotes

\n
\n

Blockquotes can also be nested...

\n
\n

...by using additional greater-than signs right next to each other...

\n
\n

...or with spaces between arrows.

\n
\n
\n
\n
" -no_ext = "

Basic markup

\n

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

\n

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
\nIt also has a newline break here!!!

\n

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

\n

__Bold without closure does not work
\n**Bold without closure does not work
\n_Emphasis without closure does not work
\n*Emphasis without closure does not work

\n

Bold without closure
\non the same line
\ncarries forward to
\nthis is text that should strike off
\nmultiple consecutive lines

\n

Bold without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Horizontal Rules

\n
\n

Above is a horizontal rule using hyphens.
\nThis is text that should strike off

\n
\n

Above is a horizontal rule using asterisks.
\nBelow is a horizontal rule using underscores.

\n
\n

Links

\n

Here is a link to hasgeek

\n

Link to funnel with the title ‘Hasgeek’

\n

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

\n
\n

Markdown-it typography

\n

The results of the below depend on the typographer options enabled by us in the markdown-it-py parser, if typographer=True has been passed to it.

\n

The below should convert if replacements has been enabled.

\n

(c) (C) (r) (R) (tm) (TM) (p) (P) +-
\ntest.. test... test..... test?..... test!....
\n!!!!!! ???? ,, -- ---

\n

The below should convert the quotes if smartquotes has been enabled.

\n

“Smartypants, double quotes” and ‘single quotes’

\n
\n

Blockquotes

\n
\n

Blockquotes can also be nested...

\n
\n

...by using additional greater-than signs right next to each other...

\n
\n

...or with spaces between arrows.

\n
\n
\n
\n
" +basic = "

Basic markup

\n

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

\n

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
\nIt also has a newline break here!!!

\n

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

\n

__Bold without closure does not work
\n**Bold without closure does not work
\n_Emphasis without closure does not work
\n*Emphasis without closure does not work

\n

Bold without closure
\non the same line
\ncarries forward to
\nthis is text that should strike off
\nmultiple consecutive lines

\n

Bold without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Horizontal Rules

\n
\n

Above is a horizontal rule using hyphens.
\nThis is text that should strike off

\n
\n

Above is a horizontal rule using asterisks.
\nBelow is a horizontal rule using underscores.

\n
\n

Links

\n

Here is a link to hasgeek

\n

Link to funnel with the title 'Hasgeek'

\n

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

\n
\n

Markdown-it typography

\n

The results of the below depend on the typographer options enabled by us in the markdown-it-py parser, if typographer=True has been passed to it.

\n

The below should convert if replacements has been enabled.

\n

(c) (C) (r) (R) (tm) (TM) (p) (P) +-
\ntest.. test... test..... test?..... test!....
\n!!!!!! ???? ,, -- ---

\n

The below should convert the quotes if smartquotes has been enabled.

\n

"Smartypants, double quotes" and 'single quotes'

\n
\n

Blockquotes

\n
\n

Blockquotes can also be nested...

\n
\n

...by using additional greater-than signs right next to each other...

\n
\n

...or with spaces between arrows.

\n
\n
\n
\n
" +document = "

Basic markup

\n

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

\n

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
\nIt also has a newline break here!!!

\n

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

\n

__Bold without closure does not work
\n**Bold without closure does not work
\n_Emphasis without closure does not work
\n*Emphasis without closure does not work

\n

Bold without closure
\non the same line
\ncarries forward to
\nthis is text that should strike off
\nmultiple consecutive lines

\n

Bold without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Horizontal Rules

\n
\n

Above is a horizontal rule using hyphens.
\nThis is text that should strike off

\n
\n

Above is a horizontal rule using asterisks.
\nBelow is a horizontal rule using underscores.

\n
\n

Links

\n

Here is a link to hasgeek

\n

Link to funnel with the title 'Hasgeek'

\n

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

\n
\n

Markdown-it typography

\n

The results of the below depend on the typographer options enabled by us in the markdown-it-py parser, if typographer=True has been passed to it.

\n

The below should convert if replacements has been enabled.

\n

(c) (C) (r) (R) (tm) (TM) (p) (P) +-
\ntest.. test... test..... test?..... test!....
\n!!!!!! ???? ,, -- ---

\n

The below should convert the quotes if smartquotes has been enabled.

\n

"Smartypants, double quotes" and 'single quotes'

\n
\n

Blockquotes

\n
\n

Blockquotes can also be nested...

\n
\n

...by using additional greater-than signs right next to each other...

\n
\n

...or with spaces between arrows.

\n
\n
\n
\n
" diff --git a/tests/data/markdown/code.toml b/tests/data/markdown/code.toml index c70e7e6f0..14464fd8e 100644 --- a/tests/data/markdown/code.toml +++ b/tests/data/markdown/code.toml @@ -61,8 +61,8 @@ Isn't that **fantastic**! """ [config] -configs = [ "", "no_ext",] +profiles = [ "basic", "document",] [expected_output] -"" = "

Code

\n

Inline code

\n

Indented code

\n
// Some comments\nline 1 of code\nline 2 of code\nline 3 of code\n
\n

Block code “fences”

\n
Sample text here...\nIt is a sample text that has multiple lines\n
\n

Syntax highlighting

\n

Javascript

\n
var foo = function (bar) {\n  return bar++;\n};\n\nconsole.log(foo(5));\nconsole.log('`This should be printed`');\n
\n
\n

Javascript can be highlighted by using either of the two keywords js and javascript

\n
\n

Python

\n
import os\nfrom funnel.utils.markdown import DATAROOT, markdown\n\nif os.file.path.exists(\n    os.file.path.join(\n        DATAROOT,\n        'file',\n        'path'\n    )\n):\n    markdown('# I can output ``` also with a \\!')\n
\n

Markdown

\n
*I can also type markdown code blocks.*\nIsn't that **fantastic**!\n\n- This is a list\n  - Just testing\n  - this out\n\n[hasgeek](https://hasgeek.com)\n
" -no_ext = "

Code

\n

Inline code

\n

Indented code

\n
// Some comments\nline 1 of code\nline 2 of code\nline 3 of code\n
\n

Block code “fences”

\n
Sample text here...\nIt is a sample text that has multiple lines\n
\n

Syntax highlighting

\n

Javascript

\n
var foo = function (bar) {\n  return bar++;\n};\n\nconsole.log(foo(5));\nconsole.log('`This should be printed`');\n
\n
\n

Javascript can be highlighted by using either of the two keywords js and javascript

\n
\n

Python

\n
import os\nfrom funnel.utils.markdown import DATAROOT, markdown\n\nif os.file.path.exists(\n    os.file.path.join(\n        DATAROOT,\n        'file',\n        'path'\n    )\n):\n    markdown('# I can output ``` also with a \\!')\n
\n

Markdown

\n
*I can also type markdown code blocks.*\nIsn't that **fantastic**!\n\n- This is a list\n  - Just testing\n  - this out\n\n[hasgeek](https://hasgeek.com)\n
" +basic = "

Code

\n

Inline code

\n

Indented code

\n
// Some comments\nline 1 of code\nline 2 of code\nline 3 of code\n
\n

Block code "fences"

\n
Sample text here...\nIt is a sample text that has multiple lines\n
\n

Syntax highlighting

\n

Javascript

\n
var foo = function (bar) {\n  return bar++;\n};\n\nconsole.log(foo(5));\nconsole.log('`This should be printed`');\n
\n
\n

Javascript can be highlighted by using either of the two keywords js and javascript

\n
\n

Python

\n
import os\nfrom funnel.utils.markdown import DATAROOT, markdown\n\nif os.file.path.exists(\n    os.file.path.join(\n        DATAROOT,\n        'file',\n        'path'\n    )\n):\n    markdown('# I can output ``` also with a \\!')\n
\n

Markdown

\n
*I can also type markdown code blocks.*\nIsn't that **fantastic**!\n\n- This is a list\n  - Just testing\n  - this out\n\n[hasgeek](https://hasgeek.com)\n
" +document = "

Code

\n

Inline code

\n

Indented code

\n
// Some comments\nline 1 of code\nline 2 of code\nline 3 of code\n
\n

Block code "fences"

\n
Sample text here...\nIt is a sample text that has multiple lines\n
\n

Syntax highlighting

\n

Javascript

\n
var foo = function (bar) {\n  return bar++;\n};\n\nconsole.log(foo(5));\nconsole.log('`This should be printed`');\n
\n
\n

Javascript can be highlighted by using either of the two keywords js and javascript

\n
\n

Python

\n
import os\nfrom funnel.utils.markdown import DATAROOT, markdown\n\nif os.file.path.exists(\n    os.file.path.join(\n        DATAROOT,\n        'file',\n        'path'\n    )\n):\n    markdown('# I can output ``` also with a \\!')\n
\n

Markdown

\n
*I can also type markdown code blocks.*\nIsn't that **fantastic**!\n\n- This is a list\n  - Just testing\n  - this out\n\n[hasgeek](https://hasgeek.com)\n
" diff --git a/tests/data/markdown/ext_ins.toml b/tests/data/markdown/ext_ins.toml.disabled similarity index 99% rename from tests/data/markdown/ext_ins.toml rename to tests/data/markdown/ext_ins.toml.disabled index 43979b66f..92582ab99 100644 --- a/tests/data/markdown/ext_ins.toml +++ b/tests/data/markdown/ext_ins.toml.disabled @@ -81,7 +81,7 @@ end_ """ [config] -configs = [ "", "no_ext",] +profiles = [ "basic", "document",] [config.extra_configs.ins] extensions = [ "ins",] diff --git a/tests/data/markdown/footnotes.toml b/tests/data/markdown/footnotes.toml index 406f742b2..97cd3271f 100644 --- a/tests/data/markdown/footnotes.toml +++ b/tests/data/markdown/footnotes.toml @@ -23,12 +23,19 @@ This is some more random text to test whether the footnotes are placed after thi """ [config] -configs = [ "", "no_ext",] - -[config.extra_configs.footnotes] -extensions = [ "footnote",] +profiles = [ "basic", "document",] + +[config.custom_profiles.footnotes] +args_config = "default" +plugins = ["footnote"] +args = ["default", {html = false, linkify = true, typographer = true, breaks = true}] +[config.custom_profiles.footnotes.args_options_update] +html = false +linkify = true +typographer = true +breaks = true [expected_output] -"" = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[1].

\n

Footnote 2 link[2].

\n

Inline footnote[3] definition.

\n

Duplicated footnote reference[2:1].

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

\n
\n
\n
    \n

  1. Footnote can have markup

    \n

    and multiple paragraphs. ↩︎

    \n
    2. \n

  2. Footnote text. ↩︎ ↩︎

    \n
    3. \n

  3. Text of inline footnote ↩︎

    \n
    4. \n
\n
" -no_ext = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[^first].

\n

Footnote 2 link[^second].

\n

Inline footnote^[Text of inline footnote] definition.

\n

Duplicated footnote reference[^second].

\n

[^first]: Footnote can have markup

\n
and multiple paragraphs.\n
\n

[^second]: Footnote text.

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

" -footnotes = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[1].

\n

Footnote 2 link[2].

\n

Inline footnote[3] definition.

\n

Duplicated footnote reference[2:1].

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

\n
\n
\n
    \n

  1. Footnote can have markup

    \n

    and multiple paragraphs. ↩︎

    \n
    2. \n

  2. Footnote text. ↩︎ ↩︎

    \n
    3. \n

  3. Text of inline footnote ↩︎

    \n
    4. \n
\n
" +basic = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[^first].

\n

Footnote 2 link[^second].

\n

Inline footnote^[Text of inline footnote] definition.

\n

Duplicated footnote reference[^second].

\n

[^first]: Footnote can have markup

\n
and multiple paragraphs.\n
\n

[^second]: Footnote text.

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

" +document = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[1].

\n

Footnote 2 link[2].

\n

Inline footnote[3] definition.

\n

Duplicated footnote reference[2:1].

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

\n
\n
\n
    \n

  1. Footnote can have markup

    \n

    and multiple paragraphs. ↩︎

    \n
    2. \n

  2. Footnote text. ↩︎ ↩︎

    \n
    3. \n

  3. Text of inline footnote ↩︎

    \n
    4. \n
\n
" +footnotes = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[1].

\n

Footnote 2 link[2].

\n

Inline footnote[3] definition.

\n

Duplicated footnote reference[2:1].

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

\n
\n
\n
    \n

  1. Footnote can have markup

    \n

    and multiple paragraphs. ↩︎

    \n
    2. \n

  2. Footnote text. ↩︎ ↩︎

    \n
    3. \n

  3. Text of inline footnote ↩︎

    \n
    4. \n
\n
" diff --git a/tests/data/markdown/headings.toml b/tests/data/markdown/headings.toml index 235df5711..167570deb 100644 --- a/tests/data/markdown/headings.toml +++ b/tests/data/markdown/headings.toml @@ -25,12 +25,19 @@ Text with 2 or more hyphens below it converts to H2 """ [config] -configs = [ "", "no_ext",] +profiles = [ "basic", "document",] -[config.extra_configs.heading_anchors] -extensions = [ "heading_anchors",] +[config.custom_profiles.heading_anchors] +args_config = "default" +plugins = [ "heading_anchors",] +args = ["default", {html = false, linkify = true, typographer = true, breaks = true}] +[config.custom_profiles.heading_anchors.args_options_update] +html = false +linkify = true +typographer = true +breaks = true [expected_output] -"" = "

Using the heading-anchors plugin with it’s default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" -no_ext = "

Using the heading-anchors plugin with it’s default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" -heading_anchors = "

Using the heading-anchors plugin with it’s default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" +basic = "

Using the heading-anchors plugin with it's default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" +document = "

Using the heading-anchors plugin with it's default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" +heading_anchors = "

Using the heading-anchors plugin with it’s default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" diff --git a/tests/data/markdown/images.toml b/tests/data/markdown/images.toml index 08e98662f..3cb85559f 100644 --- a/tests/data/markdown/images.toml +++ b/tests/data/markdown/images.toml @@ -22,8 +22,8 @@ With a reference later in the document defining the URL location. """ [config] -configs = [ "", "no_ext",] +profiles = [ "basic", "document",] [expected_output] -"" = "

Images

\n

\"Logo\"
\n\"Logo\" Images stay inline within the same block after a new line

\n

\"The

\n

Like links, images also have a footnote style syntax

\n

\"Alt:
\n\"Alt:
\n\"Alt:

\n

With a reference later in the document defining the URL location.

" -no_ext = "

Images

\n

\"Logo\"
\n\"Logo\" Images stay inline within the same block after a new line

\n

\"The

\n

Like links, images also have a footnote style syntax

\n

\"Alt:
\n\"Alt:
\n\"Alt:

\n

With a reference later in the document defining the URL location.

" +basic = "

Images

\n

\"Logo\"
\n\"Logo\" Images stay inline within the same block after a new line

\n

\"The

\n

Like links, images also have a footnote style syntax

\n

\"Alt:
\n\"Alt:
\n\"Alt:

\n

With a reference later in the document defining the URL location.

" +document = "

Images

\n

\"Logo\"
\n\"Logo\" Images stay inline within the same block after a new line

\n

\"The

\n

Like links, images also have a footnote style syntax

\n

\"Alt:
\n\"Alt:
\n\"Alt:

\n

With a reference later in the document defining the URL location.

" diff --git a/tests/data/markdown/links.toml b/tests/data/markdown/links.toml index a66874962..4079302bc 100644 --- a/tests/data/markdown/links.toml +++ b/tests/data/markdown/links.toml @@ -61,8 +61,8 @@ _http://danlec_@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoA """ [config] -configs = [ "", "no_ext",] +profiles = [ "basic", "document",] [expected_output] -"" = "

Links

\n

Hasgeek
\nHasgeek TV

\n

Autoconverted links

\n

https://github.com/nodeca/pica
\nhttp://twitter.com/hasgeek

\n

Footnote style syntax

\n

Hasgeek

\n

Links can have a footnote style syntax, where the links can be defined later. This is helpful in the case of a repetitive URL that needs to be linked to.

\n

Unsafe links

\n

These are links that should not convert (source)

\n

[a](javascript:prompt(document.cookie))
\n[a](j a v a s c r i p t:prompt(document.cookie))
\n![a](javascript:prompt(document.cookie))
\n<javascript:prompt(document.cookie)>
\n<&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29>
\n![a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\na
\n![a’"`onerror=prompt(document.cookie)](x)
\n[citelol]: (javascript:prompt(document.cookie))
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[test](javascript://%0d%0aprompt(1))
\n[test](javascript://%0d%0aprompt(1);com)
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[notmalicious](javascript://%0d%0awindow.onerror=alert;throw%20document.cookie)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[clickme](vbscript:alert(document.domain))
\nhttp://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
\ntext
\n[a](javascript:this;alert(1))
\n[a](javascript:this;alert(1))
\n[a](Javascript:alert(1))
\n[a](Javas%26%2399;ript:alert(1))
\n[a](javascript:alert�(1))
\n[a](javascript:confirm(1)
\n[a](javascript://www.google.com%0Aprompt(1))
\n[a](javascript://%0d%0aconfirm(1);com)
\n[a](javascript:window.onerror=confirm;throw%201)
\n[a](x01javascript:alert(document.domain))
\n[a](javascript://www.google.com%0Aalert(1))
\na
\n[a](JaVaScRiPt:alert(1))
\n\"a\"
\n\"a\"
\n</http://<?php><\\h1>script:scriptconfirm(2)
\nXSS
\n[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
\n[ ](http://a?p=[[/onclick=alert(0) .]])
\n[a](javascript:new%20Function`al\\ert`1``;)

" -no_ext = "

Links

\n

Hasgeek
\nHasgeek TV

\n

Autoconverted links

\n

https://github.com/nodeca/pica
\nhttp://twitter.com/hasgeek

\n

Footnote style syntax

\n

Hasgeek

\n

Links can have a footnote style syntax, where the links can be defined later. This is helpful in the case of a repetitive URL that needs to be linked to.

\n

Unsafe links

\n

These are links that should not convert (source)

\n

[a](javascript:prompt(document.cookie))
\n[a](j a v a s c r i p t:prompt(document.cookie))
\n![a](javascript:prompt(document.cookie))
\n<javascript:prompt(document.cookie)>
\n<&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29>
\n![a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\na
\n![a’"`onerror=prompt(document.cookie)](x)
\n[citelol]: (javascript:prompt(document.cookie))
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[test](javascript://%0d%0aprompt(1))
\n[test](javascript://%0d%0aprompt(1);com)
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[notmalicious](javascript://%0d%0awindow.onerror=alert;throw%20document.cookie)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[clickme](vbscript:alert(document.domain))
\nhttp://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
\ntext
\n[a](javascript:this;alert(1))
\n[a](javascript:this;alert(1))
\n[a](Javascript:alert(1))
\n[a](Javas%26%2399;ript:alert(1))
\n[a](javascript:alert�(1))
\n[a](javascript:confirm(1)
\n[a](javascript://www.google.com%0Aprompt(1))
\n[a](javascript://%0d%0aconfirm(1);com)
\n[a](javascript:window.onerror=confirm;throw%201)
\n[a](x01javascript:alert(document.domain))
\n[a](javascript://www.google.com%0Aalert(1))
\na
\n[a](JaVaScRiPt:alert(1))
\n\"a\"
\n\"a\"
\n</http://<?php><\\h1>script:scriptconfirm(2)
\nXSS
\n[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
\n[ ](http://a?p=[[/onclick=alert(0) .]])
\n[a](javascript:new%20Function`al\\ert`1``;)

" +basic = "

Links

\n

Hasgeek
\nHasgeek TV

\n

Autoconverted links

\n

https://github.com/nodeca/pica
\nhttp://twitter.com/hasgeek

\n

Footnote style syntax

\n

Hasgeek

\n

Links can have a footnote style syntax, where the links can be defined later. This is helpful in the case of a repetitive URL that needs to be linked to.

\n

Unsafe links

\n

These are links that should not convert (source)

\n

[a](javascript:prompt(document.cookie))
\n[a](j a v a s c r i p t:prompt(document.cookie))
\n![a](javascript:prompt(document.cookie))
\n<javascript:prompt(document.cookie)>
\n<&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29>
\n![a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\na
\n![a'"`onerror=prompt(document.cookie)](x)
\n[citelol]: (javascript:prompt(document.cookie))
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[test](javascript://%0d%0aprompt(1))
\n[test](javascript://%0d%0aprompt(1);com)
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[notmalicious](javascript://%0d%0awindow.onerror=alert;throw%20document.cookie)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[clickme](vbscript:alert(document.domain))
\nhttp://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
\ntext
\n[a](javascript:this;alert(1))
\n[a](javascript:this;alert(1))
\n[a](Javascript:alert(1))
\n[a](Javas%26%2399;ript:alert(1))
\n[a](javascript:alert�(1))
\n[a](javascript:confirm(1)
\n[a](javascript://www.google.com%0Aprompt(1))
\n[a](javascript://%0d%0aconfirm(1);com)
\n[a](javascript:window.onerror=confirm;throw%201)
\n[a](x01javascript:alert(document.domain))
\n[a](javascript://www.google.com%0Aalert(1))
\na
\n[a](JaVaScRiPt:alert(1))
\n\"a\"
\n\"a\"
\n</http://<?php><\\h1>script:scriptconfirm(2)
\nXSS
\n[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
\n[ ](http://a?p=[[/onclick=alert(0) .]])
\n[a](javascript:new%20Function`al\\ert`1``;)

" +document = "

Links

\n

Hasgeek
\nHasgeek TV

\n

Autoconverted links

\n

https://github.com/nodeca/pica
\nhttp://twitter.com/hasgeek

\n

Footnote style syntax

\n

Hasgeek

\n

Links can have a footnote style syntax, where the links can be defined later. This is helpful in the case of a repetitive URL that needs to be linked to.

\n

Unsafe links

\n

These are links that should not convert (source)

\n

[a](javascript:prompt(document.cookie))
\n[a](j a v a s c r i p t:prompt(document.cookie))
\n![a](javascript:prompt(document.cookie))
\n<javascript:prompt(document.cookie)>
\n<&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29>
\n![a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\na
\n![a'"`onerror=prompt(document.cookie)](x)
\n[citelol]: (javascript:prompt(document.cookie))
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[test](javascript://%0d%0aprompt(1))
\n[test](javascript://%0d%0aprompt(1);com)
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[notmalicious](javascript://%0d%0awindow.onerror=alert;throw%20document.cookie)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[clickme](vbscript:alert(document.domain))
\nhttp://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
\ntext
\n[a](javascript:this;alert(1))
\n[a](javascript:this;alert(1))
\n[a](Javascript:alert(1))
\n[a](Javas%26%2399;ript:alert(1))
\n[a](javascript:alert�(1))
\n[a](javascript:confirm(1)
\n[a](javascript://www.google.com%0Aprompt(1))
\n[a](javascript://%0d%0aconfirm(1);com)
\n[a](javascript:window.onerror=confirm;throw%201)
\n[a](x01javascript:alert(document.domain))
\n[a](javascript://www.google.com%0Aalert(1))
\na
\n[a](JaVaScRiPt:alert(1))
\n\"a\"
\n\"a\"
\n</http://<?php><\\h1>script:scriptconfirm(2)
\nXSS
\n[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
\n[ ](http://a?p=[[/onclick=alert(0) .]])
\n[a](javascript:new%20Function`al\\ert`1``;)

" diff --git a/tests/data/markdown/lists.toml b/tests/data/markdown/lists.toml index 0565b7ce9..b59f25d0c 100644 --- a/tests/data/markdown/lists.toml +++ b/tests/data/markdown/lists.toml @@ -34,8 +34,8 @@ Start numbering with offset: """ [config] -configs = [ "", "no_ext",] +profiles = [ "basic", "document",] [expected_output] -"" = "

Lists

\n

Unordered

\n
    \n
  • Create a list by starting a line with +, -, or *
    • \n
  • Sub-lists are made by indenting 2 spaces:\n
      \n
    • Marker character change forces new list start:\n
        \n
      • Ac tristique libero volutpat at
        • \n
      \n
        \n
      • Facilisis in pretium nisl aliquet
        • \n
      \n
        \n
      • Nulla volutpat aliquam velit
        • \n
      \n
      • \n
    \n
    • \n
  • Very easy!
    • \n
\n

Ordered

\n
    \n
  1. \n

    Lorem ipsum dolor sit amet

    \n
    2. \n
  2. \n

    Consectetur adipiscing elit

    \n
    3. \n
  3. \n

    Integer molestie lorem at massa

    \n
    4. \n
  4. \n

    You can use sequential numbers...

    \n
    5. \n
  5. \n

    ...or keep all the numbers as 1.

    \n
    6. \n
\n
    \n
  1. You can start a new list by switching between ) and .
    2. \n
  2. Skipping numbers does not have any effect.
    3. \n
\n

Start numbering with offset:

\n
    \n
  1. foo
    2. \n
  2. bar
    3. \n
\n
\n

2022. If you do not want to render a list, use \\ to escape the .

" -no_ext = "

Lists

\n

Unordered

\n
    \n
  • Create a list by starting a line with +, -, or *
    • \n
  • Sub-lists are made by indenting 2 spaces:\n
      \n
    • Marker character change forces new list start:\n
        \n
      • Ac tristique libero volutpat at
        • \n
      \n
        \n
      • Facilisis in pretium nisl aliquet
        • \n
      \n
        \n
      • Nulla volutpat aliquam velit
        • \n
      \n
      • \n
    \n
    • \n
  • Very easy!
    • \n
\n

Ordered

\n
    \n
  1. \n

    Lorem ipsum dolor sit amet

    \n
    2. \n
  2. \n

    Consectetur adipiscing elit

    \n
    3. \n
  3. \n

    Integer molestie lorem at massa

    \n
    4. \n
  4. \n

    You can use sequential numbers...

    \n
    5. \n
  5. \n

    ...or keep all the numbers as 1.

    \n
    6. \n
\n
    \n
  1. You can start a new list by switching between ) and .
    2. \n
  2. Skipping numbers does not have any effect.
    3. \n
\n

Start numbering with offset:

\n
    \n
  1. foo
    2. \n
  2. bar
    3. \n
\n
\n

2022. If you do not want to render a list, use \\ to escape the .

" +basic = "

Lists

\n

Unordered

\n
    \n
  • Create a list by starting a line with +, -, or *
    • \n
  • Sub-lists are made by indenting 2 spaces:\n
      \n
    • Marker character change forces new list start:\n
        \n
      • Ac tristique libero volutpat at
        • \n
      \n
        \n
      • Facilisis in pretium nisl aliquet
        • \n
      \n
        \n
      • Nulla volutpat aliquam velit
        • \n
      \n
      • \n
    \n
    • \n
  • Very easy!
    • \n
\n

Ordered

\n
    \n
  1. \n

    Lorem ipsum dolor sit amet

    \n
    2. \n
  2. \n

    Consectetur adipiscing elit

    \n
    3. \n
  3. \n

    Integer molestie lorem at massa

    \n
    4. \n
  4. \n

    You can use sequential numbers...

    \n
    5. \n
  5. \n

    ...or keep all the numbers as 1.

    \n
    6. \n
\n
    \n
  1. You can start a new list by switching between ) and .
    2. \n
  2. Skipping numbers does not have any effect.
    3. \n
\n

Start numbering with offset:

\n
    \n
  1. foo
    2. \n
  2. bar
    3. \n
\n
\n

2022. If you do not want to render a list, use \\ to escape the .

" +document = "

Lists

\n

Unordered

\n
    \n
  • Create a list by starting a line with +, -, or *
    • \n
  • Sub-lists are made by indenting 2 spaces:\n
      \n
    • Marker character change forces new list start:\n
        \n
      • Ac tristique libero volutpat at
        • \n
      \n
        \n
      • Facilisis in pretium nisl aliquet
        • \n
      \n
        \n
      • Nulla volutpat aliquam velit
        • \n
      \n
      • \n
    \n
    • \n
  • Very easy!
    • \n
\n

Ordered

\n
    \n
  1. \n

    Lorem ipsum dolor sit amet

    \n
    2. \n
  2. \n

    Consectetur adipiscing elit

    \n
    3. \n
  3. \n

    Integer molestie lorem at massa

    \n
    4. \n
  4. \n

    You can use sequential numbers...

    \n
    5. \n
  5. \n

    ...or keep all the numbers as 1.

    \n
    6. \n
\n
    \n
  1. You can start a new list by switching between ) and .
    2. \n
  2. Skipping numbers does not have any effect.
    3. \n
\n

Start numbering with offset:

\n
    \n
  1. foo
    2. \n
  2. bar
    3. \n
\n
\n

2022. If you do not want to render a list, use \\ to escape the .

" diff --git a/tests/data/markdown/tables.toml b/tests/data/markdown/tables.toml index eccf70332..944f58cdd 100644 --- a/tests/data/markdown/tables.toml +++ b/tests/data/markdown/tables.toml @@ -18,8 +18,8 @@ markdown = """ """ [config] -configs = [ "", "no_ext",] +profiles = [ "basic", "document",] [expected_output] -"" = "

Tables

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
\n

Right aligned columns

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
" -no_ext = "

Tables

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
\n

Right aligned columns

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
" +basic = "

Tables

\n

| Option | Description |
\n| ------ | ----------- |
\n| data | path to data files to supply the data that will be passed into templates. |
\n| engine | engine to be used for processing templates. Handlebars is the default. |
\n| ext | extension to be used for dest files. |

\n

Right aligned columns

\n

| Option | Description |
\n| ------:| -----------:|
\n| data | path to data files to supply the data that will be passed into templates. |
\n| engine | engine to be used for processing templates. Handlebars is the default. |
\n| ext | extension to be used for dest files. |

" +document = "

Tables

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
\n

Right aligned columns

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
" diff --git a/tests/data/markdown/template.html b/tests/data/markdown/template.html index 5653d9023..84793fd97 100644 --- a/tests/data/markdown/template.html +++ b/tests/data/markdown/template.html @@ -129,8 +129,8 @@

Markdown Tests: Expected Output compared to Output

▶ ▼ - Test file: , Config key: - + Test file: , Configuration profile: +
Configuration:
diff --git a/tests/unit/utils/test_markdown.py b/tests/unit/utils/test_markdown.py index baee142fb..c739776fa 100644 --- a/tests/unit/utils/test_markdown.py +++ b/tests/unit/utils/test_markdown.py @@ -14,7 +14,7 @@ import tomlkit from funnel.utils import markdown -from funnel.utils.markdown.helpers import MD_CONFIGS +from funnel.utils.markdown.profiles import default_markdown_options, profiles DATA_ROOT = os.path.abspath(os.path.join('tests', 'data', 'markdown')) @@ -36,43 +36,40 @@ def load_md_cases() -> CasesType: return cases -def blank_to_none(_id: str): - """Replace blank string with None.""" - return None if _id == "" else _id - - -def none_to_blank(_id: Optional[str]): - """Replace None with a blank string.""" - return "" if _id is None else str(_id) - - -def get_case_configs(case: CaseType) -> Dict[str, Any]: - """Return dict with key-value of configs for the provided test case.""" - case_configs: Dict[str, Any] = {} - configs = copy(case['config']['configs']) - md_configs = deepcopy(MD_CONFIGS) - if 'extra_configs' in case['config']: - for c in case['config']['extra_configs']: - if c not in md_configs: - md_configs[c] = case['config']['extra_configs'][c] - if c not in configs: - configs.append(c) - for c in configs: - c = blank_to_none(c) - if c in md_configs: - case_configs[c] = md_configs[c] +def get_case_profiles(case: CaseType) -> Dict[str, Any]: + """Return dict with key-value of profiles for the provided test case.""" + profiles_out: Dict[str, Any] = {} + case_profiles = copy(case['config']['profiles']) + _profiles = deepcopy(profiles) + if 'custom_profiles' in case['config']: + for p in case['config']['custom_profiles']: + if p not in _profiles: + _profiles[p] = case['config']['custom_profiles'][p] + if _profiles[p]['args_options_update'] is False: + _profiles[p]['args_options_update'] = default_markdown_options + _profiles[p]['args'] = ( + _profiles[p]['args_config'], + _profiles[p]['args_options_update'], + ) + if p not in case_profiles: + case_profiles.append(p) + for p in case_profiles: + if p in _profiles: + profiles_out[p] = _profiles[p] else: - case_configs[c] = None - return case_configs + profiles_out[p] = None + return profiles_out -def get_md(case: CaseType, config: Optional[Dict[str, Any]]): - """Parse a markdown test case for given configuration.""" - if config is None: - return markdown("This configuration does not exist in `MD_CONFIG`.").__str__() +def get_md(case: CaseType, profile: Optional[Dict[str, Any]]): + """Parse a markdown test case for given configuration profile.""" + if profile is None: + return markdown( + 'This configuration profile has not been specified.', 'basic' + ).__str__() return ( markdown( # pylint: disable=unnecessary-dunder-call - case['data']['markdown'], **config + case['data']['markdown'], profile ) .__str__() .lstrip('\n\r') @@ -80,14 +77,16 @@ def get_md(case: CaseType, config: Optional[Dict[str, Any]]): ) -def update_md_case_results(cases: CasesType) -> None: - """Update cases object with expected result for each case-config combination.""" +def update_expected_case_output(cases: CasesType) -> None: + """Update cases object with expected output for each case-profile combination.""" for case_id in cases: case = cases[case_id] - configs = get_case_configs(case) + _profiles = get_case_profiles(case) case['expected_output'] = {} - for config_id, config in configs.items(): - case['expected_output'][none_to_blank(config_id)] = get_md(case, config) + for profile_id, _profile in _profiles.items(): + case['expected_output'][profile_id] = get_md( + case, profiles[profile_id] if profile_id in profiles else _profile + ) def dump_md_cases(cases: CasesType) -> None: @@ -101,7 +100,7 @@ def dump_md_cases(cases: CasesType) -> None: def update_test_data() -> None: """Update test data after changes made to test cases and/or configurations.""" cases = load_md_cases() - update_md_case_results(cases) + update_expected_case_output(cases) dump_md_cases(cases) @@ -125,38 +124,44 @@ def update_case_output( template: BeautifulSoup, case_id: str, case: CaseType, - config_id: str, - config: Dict[str, Any], + profile_id: str, + profile: Dict[str, Any], output: str, ) -> None: """Update & return case template with output for provided case-configuration.""" + profile = deepcopy(profile) if 'output' not in case: case['output'] = {} - case['output'][none_to_blank(config_id)] = output + case['output'][profile_id] = output op = copy(get_case_template()) del op['id'] op.select('.filename')[0].string = case_id - op.select('.configname')[0].string = str(config_id) - op.select('.config')[0].string = json.dumps(config, indent=2) + op.select('.profile')[0].string = str(profile_id) + if 'args_config' in profile: + del profile['args_config'] + if 'args_options_update' in profile: + del profile['args_options_update'] + op.select('.config')[0].string = json.dumps(profile, indent=2) op.select('.markdown .output')[0].append(case['data']['markdown']) try: - expected_output = case['expected_output'][none_to_blank(config_id)] + expected_output = case['expected_output'][profile_id] except KeyError: expected_output = markdown( - f'Expected output for `{case_id}` config `{config_id}` ' + f'Expected output for `{case_id}` config `{profile_id}` ' 'has not been generated. Please run `make tests-data-md`' '**after evaluating other failures**.\n' '`make tests-data-md` should only be run for this after ' 'ensuring there are no unexpected mismatches/failures ' 'in the output of all cases.\n' - 'For detailed instructions check the [readme](readme.md).' + 'For detailed instructions check the [readme](readme.md).', + 'basic', ) op.select('.expected .output')[0].append( BeautifulSoup(expected_output, 'html.parser') ) op.select('.final_output .output')[0].append(BeautifulSoup(output, 'html.parser')) op['class'] = op.get('class', []) + [ - 'success' if expected_output == output and config is not None else 'failed' + 'success' if expected_output == output and profile is not None else 'failed' ] template.find('body').append(op) @@ -174,15 +179,15 @@ def get_md_test_data() -> CasesType: template = get_output_template() cases = load_md_cases() for case_id, case in cases.items(): - configs = get_case_configs(case) - for config_id, config in configs.items(): - test_output = get_md(case, config) + _profiles = get_case_profiles(case) + for profile_id, profile in _profiles.items(): + test_output = get_md(case, profile) update_case_output( template, case_id, case, - config_id, - config, + profile_id, + profile, test_output, ) dump_md_output(template) @@ -192,23 +197,23 @@ def get_md_test_data() -> CasesType: def get_md_test_dataset() -> List[Tuple[str, str]]: """Return testcase datasets.""" return [ - (case_id, none_to_blank(config_id)) + (case_id, profile_id) for (case_id, case) in load_md_cases().items() - for config_id in get_case_configs(case) + for profile_id in get_case_profiles(case) ] -def get_md_test_output(case_id: str, config_id: str) -> Tuple[str, str]: +def get_md_test_output(case_id: str, profile_id: str) -> Tuple[str, str]: """Return expected output and final output for quoted case-config combination.""" cases = get_md_test_data() try: return ( - cases[case_id]['expected_output'][none_to_blank(config_id)], - cases[case_id]['output'][none_to_blank(config_id)], + cases[case_id]['expected_output'][profile_id], + cases[case_id]['output'][profile_id], ) except KeyError: return ( - f'Expected output for "{case_id}" config "{config_id}" has not been generated. ' + f'Expected output for "{case_id}" config "{profile_id}" has not been generated. ' 'Please run "make tests-data-md" ' 'after evaluating other failures.\n' '"make tests-data-md" should only be run for this after ' @@ -216,13 +221,13 @@ def get_md_test_output(case_id: str, config_id: str) -> Tuple[str, str]: 'in the output of all cases.\n', 'For detailed instructions check "tests/data/markdown/readme.md". \n' + ( - cases[case_id]['output'][none_to_blank(config_id)] - if len(cases[case_id]['output'][none_to_blank(config_id)]) <= 160 + cases[case_id]['output'][profile_id] + if len(cases[case_id]['output'][profile_id]) <= 160 else '\n'.join( [ - cases[case_id]['output'][none_to_blank(config_id)][:80], + cases[case_id]['output'][profile_id][:80], '...', - cases[case_id]['output'][none_to_blank(config_id)][-80:], + cases[case_id]['output'][profile_id][-80:], ] ) ), @@ -230,11 +235,13 @@ def get_md_test_output(case_id: str, config_id: str) -> Tuple[str, str]: def test_markdown_none() -> None: - assert markdown(None) is None + assert markdown(None, 'basic') is None + assert markdown(None, 'document') is None def test_markdown_blank() -> None: - assert markdown('') == Markup('') + assert markdown('', 'basic') == Markup('') + assert markdown('', 'document') == Markup('') @pytest.mark.update_markdown_data() @@ -248,16 +255,16 @@ def test_markdown_update_output(pytestconfig): @pytest.mark.parametrize( ( 'case_id', - 'config_id', + 'profile_id', ), get_md_test_dataset(), ) -def test_markdown_dataset(case_id: str, config_id: str) -> None: - (expected_output, output) = get_md_test_output(case_id, config_id) +def test_markdown_dataset(case_id: str, profile_id: str) -> None: + (expected_output, output) = get_md_test_output(case_id, profile_id) cases = load_md_cases() - configs = get_case_configs(cases[case_id]) - if configs[blank_to_none(config_id)] is None or expected_output != output: - if configs[blank_to_none(config_id)] is None: + _profiles = get_case_profiles(cases[case_id]) + if _profiles[profile_id] is None or expected_output != output: + if _profiles[profile_id] is None: msg = [output] else: difference = context_diff(expected_output.split('\n'), output.split('\n')) @@ -268,7 +275,7 @@ def test_markdown_dataset(case_id: str, config_id: str) -> None: pytest.fail( '\n'.join( [ - f'Markdown output failed. File: {case_id}, Config key: {config_id}.', + f'Markdown output failed. File: {case_id}, Config key: {profile_id}.', 'Please check tests/data/markdown/output.html for detailed output comparision', ] + msg From 1e05ab15888c624826167e76a60fe42e64ff9a28 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 30 Sep 2022 02:31:33 +0530 Subject: [PATCH 08/72] Effort to clean up markdown tests by segregating and moving all debug logic out of the main tests. #1490 --- tests/conftest.py | 15 + tests/unit/utils/test_markdown.py | 479 +++++++++++++++--------------- 2 files changed, 257 insertions(+), 237 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ad185d0b7..c0303dff2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ from __future__ import annotations from datetime import datetime, timezone +from difflib import unified_diff from types import MethodType, SimpleNamespace import re import shutil @@ -1666,3 +1667,17 @@ def new_proposal(models, db_session, new_user, new_project): db_session.add(proposal) db_session.commit() return proposal + + +@pytest.fixture() +def unified_diff_output(): + def func(left, right): + if left != right: + difference = unified_diff(left.split('\n'), right.split('\n')) + msg = [] + for line in difference: + if not line.startswith(' '): + msg.append(line) + pytest.fail('\n'.join(msg), pytrace=False) + + return func diff --git a/tests/unit/utils/test_markdown.py b/tests/unit/utils/test_markdown.py index c739776fa..bd7915cc3 100644 --- a/tests/unit/utils/test_markdown.py +++ b/tests/unit/utils/test_markdown.py @@ -2,11 +2,10 @@ from copy import copy, deepcopy from datetime import datetime -from difflib import context_diff from functools import lru_cache -from typing import Any, Dict, List, Optional, Tuple +from pathlib import Path +from typing import Any, Callable, Dict, List, Tuple, Union import json -import os from bs4 import BeautifulSoup from markupsafe import Markup @@ -16,222 +15,242 @@ from funnel.utils import markdown from funnel.utils.markdown.profiles import default_markdown_options, profiles -DATA_ROOT = os.path.abspath(os.path.join('tests', 'data', 'markdown')) - -CaseType = Dict[str, Any] -CasesType = Dict[str, CaseType] - - -@lru_cache() -def load_md_cases() -> CasesType: - """Load test cases for the markdown parser from .toml files.""" - cases: CasesType = {} - files = os.listdir(DATA_ROOT) - files.sort() - for file in files: - if file.endswith('.toml'): - with open(os.path.join(DATA_ROOT, file), encoding='utf-8') as f: - cases[file] = tomlkit.load(f) - f.close() - return cases - - -def get_case_profiles(case: CaseType) -> Dict[str, Any]: - """Return dict with key-value of profiles for the provided test case.""" - profiles_out: Dict[str, Any] = {} - case_profiles = copy(case['config']['profiles']) - _profiles = deepcopy(profiles) - if 'custom_profiles' in case['config']: - for p in case['config']['custom_profiles']: - if p not in _profiles: - _profiles[p] = case['config']['custom_profiles'][p] - if _profiles[p]['args_options_update'] is False: - _profiles[p]['args_options_update'] = default_markdown_options - _profiles[p]['args'] = ( - _profiles[p]['args_config'], - _profiles[p]['args_options_update'], - ) - if p not in case_profiles: - case_profiles.append(p) - for p in case_profiles: - if p in _profiles: - profiles_out[p] = _profiles[p] - else: - profiles_out[p] = None - return profiles_out - - -def get_md(case: CaseType, profile: Optional[Dict[str, Any]]): - """Parse a markdown test case for given configuration profile.""" - if profile is None: - return markdown( - 'This configuration profile has not been specified.', 'basic' - ).__str__() - return ( - markdown( # pylint: disable=unnecessary-dunder-call - case['data']['markdown'], profile - ) - .__str__() - .lstrip('\n\r') - .rstrip(' \n\r') - ) - - -def update_expected_case_output(cases: CasesType) -> None: - """Update cases object with expected output for each case-profile combination.""" - for case_id in cases: - case = cases[case_id] - _profiles = get_case_profiles(case) - case['expected_output'] = {} - for profile_id, _profile in _profiles.items(): - case['expected_output'][profile_id] = get_md( - case, profiles[profile_id] if profile_id in profiles else _profile - ) - - -def dump_md_cases(cases: CasesType) -> None: - """Save test cases for the markdown parser to .toml files.""" - for (file, file_data) in cases.items(): - with open(os.path.join(DATA_ROOT, file), 'w', encoding='utf-8') as f: - tomlkit.dump(file_data, f) - f.close() - - -def update_test_data() -> None: - """Update test data after changes made to test cases and/or configurations.""" - cases = load_md_cases() - update_expected_case_output(cases) - dump_md_cases(cases) +def markdown_fn(fn: str) -> Callable: + dataroot: Path = Path('tests/data/markdown') + + case_type = Dict[str, Any] + cases_type = Dict[str, case_type] + + @lru_cache() + def load_md_cases() -> cases_type: + """Load test cases for the markdown parser from .toml files.""" + return { + file.name: tomlkit.loads(file.read_text()) + for file in dataroot.iterdir() + if file.suffix == '.toml' + } + + def get_case_profiles(case: case_type) -> Dict[str, Any]: + """Return dict with key-value of profiles for the provided test case.""" + profiles_out: Dict[str, Any] = {} + case_profiles = copy(case['config']['profiles']) + _profiles = deepcopy(profiles) + if 'custom_profiles' in case['config']: + for p in case['config']['custom_profiles']: + if p not in _profiles: + _profiles[p] = case['config']['custom_profiles'][p] + if _profiles[p]['args_options_update'] is False: + _profiles[p]['args_options_update'] = default_markdown_options + _profiles[p]['args'] = ( + _profiles[p]['args_config'], + _profiles[p]['args_options_update'], + ) + if p not in case_profiles: + case_profiles.append(p) + for p in case_profiles: + if p in _profiles: + profiles_out[p] = _profiles[p] + else: + profiles_out[p] = None + return profiles_out + + def get_md(case: case_type, profile: Union[str, Dict[str, Any]]): + """Parse a markdown test case for given configuration profile.""" + return ( + markdown( # pylint: disable=unnecessary-dunder-call + case['data']['markdown'], profile + ) + .__str__() + .lstrip('\n\r') + .rstrip(' \n\r') + ) -@lru_cache() -def get_output_template() -> BeautifulSoup: - """Get bs4 output template for output.html for markdown tests.""" - with open(os.path.join(DATA_ROOT, 'template.html'), encoding='utf-8') as f: - template = BeautifulSoup(f, 'html.parser') - f.close() - return template - - -@lru_cache() -def get_case_template() -> BeautifulSoup: - """Get blank bs4 template for each case to be used to update test output.""" - return get_output_template().find(id='output_template') - + def update_expected_case_output(cases: cases_type, debug: bool) -> None: + """Update cases object with expected output for each case-profile combination.""" + for case_id in cases: + case = cases[case_id] + _profiles = get_case_profiles(case) + case['expected_output'] = {} + for profile_id, _profile in _profiles.items(): + case['expected_output'][profile_id] = get_md( + case, + profiles[profile_id] if profile_id in profiles else _profile, + ) -# pylint: disable=too-many-arguments -def update_case_output( - template: BeautifulSoup, - case_id: str, - case: CaseType, - profile_id: str, - profile: Dict[str, Any], - output: str, -) -> None: - """Update & return case template with output for provided case-configuration.""" - profile = deepcopy(profile) - if 'output' not in case: - case['output'] = {} - case['output'][profile_id] = output - op = copy(get_case_template()) - del op['id'] - op.select('.filename')[0].string = case_id - op.select('.profile')[0].string = str(profile_id) - if 'args_config' in profile: - del profile['args_config'] - if 'args_options_update' in profile: - del profile['args_options_update'] - op.select('.config')[0].string = json.dumps(profile, indent=2) - op.select('.markdown .output')[0].append(case['data']['markdown']) - try: - expected_output = case['expected_output'][profile_id] - except KeyError: - expected_output = markdown( - f'Expected output for `{case_id}` config `{profile_id}` ' - 'has not been generated. Please run `make tests-data-md`' - '**after evaluating other failures**.\n' - '`make tests-data-md` should only be run for this after ' - 'ensuring there are no unexpected mismatches/failures ' - 'in the output of all cases.\n' - 'For detailed instructions check the [readme](readme.md).', - 'basic', - ) - op.select('.expected .output')[0].append( - BeautifulSoup(expected_output, 'html.parser') - ) - op.select('.final_output .output')[0].append(BeautifulSoup(output, 'html.parser')) - op['class'] = op.get('class', []) + [ - 'success' if expected_output == output and profile is not None else 'failed' - ] - template.find('body').append(op) - - -def dump_md_output(output: BeautifulSoup) -> None: - """Save test output in output.html.""" - output.find(id='generated').string = datetime.now().strftime('%d %B, %Y %H:%M:%S') - with open(os.path.join(DATA_ROOT, 'output.html'), 'w', encoding='utf-8') as f: - f.write(output.prettify()) - - -@lru_cache() -def get_md_test_data() -> CasesType: - """Get cases updated with final output alongwith test cases dataset.""" - template = get_output_template() - cases = load_md_cases() - for case_id, case in cases.items(): - _profiles = get_case_profiles(case) - for profile_id, profile in _profiles.items(): - test_output = get_md(case, profile) - update_case_output( - template, - case_id, - case, - profile_id, - profile, - test_output, + def dump_md_cases(cases: cases_type) -> None: + """Save test cases for the markdown parser to .toml files.""" + for (file, file_data) in cases.items(): + (dataroot / file).write_text(tomlkit.dumps(file_data)) + + def update_test_data(debug: bool = False) -> None: + """Update test data after changes made to test cases and/or configurations.""" + cases = load_md_cases() + update_expected_case_output(cases, debug) + dump_md_cases(cases) + + @lru_cache() + def get_output_template() -> BeautifulSoup: + """Get bs4 output template for output.html for markdown tests.""" + return BeautifulSoup((dataroot / 'template.html').read_text(), 'html.parser') + + @lru_cache() + def get_case_template() -> BeautifulSoup: + """Get blank bs4 template for each case to be used to update test output.""" + return get_output_template().find(id='output_template') + + # pylint: disable=too-many-arguments + def update_case_output( + case: case_type, + profile_id: str, + output: str, + ) -> None: + """Update case with output for provided case-configuration.""" + if 'output' not in case: + case['output'] = {} + case['output'][profile_id] = output + + def update_case_output_template( + template: BeautifulSoup, + case_id: str, + case: case_type, + profile_id: str, + profile: Dict[str, Any], + output: str, + ) -> None: + """Update case and output template with output for provided case-configuration.""" + profile = deepcopy(profile) + if 'output' not in case: + case['output'] = {} + case['output'][profile_id] = output + op = copy(get_case_template()) + del op['id'] + op.select('.filename')[0].string = case_id + op.select('.profile')[0].string = str(profile_id) + if 'args_config' in profile: + del profile['args_config'] + if 'args_options_update' in profile: + del profile['args_options_update'] + op.select('.config')[0].string = json.dumps(profile, indent=2) + op.select('.markdown .output')[0].append(case['data']['markdown']) + try: + expected_output = case['expected_output'][profile_id] + except KeyError: + expected_output = markdown( + f'Expected output for `{case_id}` config `{profile_id}` ' + 'has not been generated. Please run `make tests-data-md`' + '**after evaluating other failures**.\n' + '`make tests-data-md` should only be run for this after ' + 'ensuring there are no unexpected mismatches/failures ' + 'in the output of all cases.\n' + 'For detailed instructions check the [readme](readme.md).', + 'basic', ) - dump_md_output(template) - return cases + op.select('.expected .output')[0].append( + BeautifulSoup(expected_output, 'html.parser') + ) + op.select('.final_output .output')[0].append( + BeautifulSoup(output, 'html.parser') + ) + op['class'] = op.get('class', []) + [ + 'success' if expected_output == output and profile is not None else 'failed' + ] + template.find('body').append(op) + + def dump_md_output(output: BeautifulSoup) -> None: + """Save test output in output.html.""" + output.find(id='generated').string = datetime.now().strftime( + '%d %B, %Y %H:%M:%S' + ) + (dataroot / 'output.html').write_text(output.prettify()) + + @lru_cache() + def get_md_test_data(debug: bool) -> cases_type: + """Get cases updated with final output alongwith test cases dataset.""" + if debug: + template = get_output_template() + cases = load_md_cases() + for case_id, case in cases.items(): + _profiles = get_case_profiles(case) + for profile_id, profile in _profiles.items(): + test_output = get_md(case, profile) + if debug: + update_case_output_template( + template, + case_id, + case, + profile_id, + profile, + test_output, + ) + else: + update_case_output(case, profile_id, test_output) + if debug: + dump_md_output(template) + return cases + + def get_md_test_dataset() -> List[Tuple[str, str]]: + """Return testcase datasets.""" + return [ + (case_id, profile_id) + for (case_id, case) in load_md_cases().items() + for profile_id in get_case_profiles(case) + ] + + @lru_cache() + def get_md_test_output( + case_id: str, profile_id: str, debug: bool = False + ) -> Tuple[str, str]: + """Return expected output and final output for quoted case-config combination.""" + cases = get_md_test_data(debug) + if not debug: + expected_output = cases[case_id]['expected_output'][profile_id] + output = cases[case_id]['output'][profile_id] + else: + try: + expected_output = cases[case_id]['expected_output'][profile_id] + output = cases[case_id]['output'][profile_id] + except KeyError: + return ( + f'Expected output for "{case_id}" config "{profile_id}" has not been generated. ' + 'Please run "make tests-data-md" ' + 'after evaluating other failures.\n' + '"make tests-data-md" should only be run for this after ' + 'ensuring there are no unexpected mismatches/failures ' + 'in the output of all cases.\n', + 'For detailed instructions check "tests/data/markdown/readme.md". \n' + + ( + cases[case_id]['output'][profile_id] + if len(cases[case_id]['output'][profile_id]) <= 160 + else '\n'.join( + [ + cases[case_id]['output'][profile_id][:80], + '...', + cases[case_id]['output'][profile_id][-80:], + ] + ) + ), + ) + return (expected_output, output) + if fn == 'get_dataset': + return get_md_test_dataset + elif fn == 'get_data': + return get_md_test_output + elif fn == 'update_data': + return update_test_data + else: + return lambda x: None -def get_md_test_dataset() -> List[Tuple[str, str]]: - """Return testcase datasets.""" - return [ - (case_id, profile_id) - for (case_id, case) in load_md_cases().items() - for profile_id in get_case_profiles(case) - ] +@pytest.fixture() +def update_markdown_tests_data(): + return markdown_fn('update_data') -def get_md_test_output(case_id: str, profile_id: str) -> Tuple[str, str]: - """Return expected output and final output for quoted case-config combination.""" - cases = get_md_test_data() - try: - return ( - cases[case_id]['expected_output'][profile_id], - cases[case_id]['output'][profile_id], - ) - except KeyError: - return ( - f'Expected output for "{case_id}" config "{profile_id}" has not been generated. ' - 'Please run "make tests-data-md" ' - 'after evaluating other failures.\n' - '"make tests-data-md" should only be run for this after ' - 'ensuring there are no unexpected mismatches/failures ' - 'in the output of all cases.\n', - 'For detailed instructions check "tests/data/markdown/readme.md". \n' - + ( - cases[case_id]['output'][profile_id] - if len(cases[case_id]['output'][profile_id]) <= 160 - else '\n'.join( - [ - cases[case_id]['output'][profile_id][:80], - '...', - cases[case_id]['output'][profile_id][-80:], - ] - ) - ), - ) + +@pytest.fixture(scope='session') +def markdown_output(): + return markdown_fn('get_data') def test_markdown_none() -> None: @@ -245,11 +264,11 @@ def test_markdown_blank() -> None: @pytest.mark.update_markdown_data() -def test_markdown_update_output(pytestconfig): +def test_markdown_update_output(pytestconfig, update_markdown_tests_data): has_mark = pytestconfig.getoption('-m', default=None) == 'update_markdown_data' if not has_mark: pytest.skip('Skipping update of expected output of markdown test cases') - update_test_data() + update_markdown_tests_data(debug=False) @pytest.mark.parametrize( @@ -257,28 +276,14 @@ def test_markdown_update_output(pytestconfig): 'case_id', 'profile_id', ), - get_md_test_dataset(), + markdown_fn('get_dataset')(), ) -def test_markdown_dataset(case_id: str, profile_id: str) -> None: - (expected_output, output) = get_md_test_output(case_id, profile_id) - cases = load_md_cases() - _profiles = get_case_profiles(cases[case_id]) - if _profiles[profile_id] is None or expected_output != output: - if _profiles[profile_id] is None: - msg = [output] - else: - difference = context_diff(expected_output.split('\n'), output.split('\n')) - msg = [] - for line in difference: - if not line.startswith(' '): - msg.append(line) - pytest.fail( - '\n'.join( - [ - f'Markdown output failed. File: {case_id}, Config key: {profile_id}.', - 'Please check tests/data/markdown/output.html for detailed output comparision', - ] - + msg - ), - pytrace=False, - ) +def test_markdown_dataset( + case_id: str, profile_id: str, markdown_output, unified_diff_output +) -> None: + debug = False + (expected_output, output) = markdown_output(case_id, profile_id, debug=debug) + if debug: + unified_diff_output(expected_output, output) + else: + assert expected_output == output From e01c62015dcd586f54605ab06972ea07bdcafe75 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 11 Oct 2022 12:50:58 +0530 Subject: [PATCH 09/72] Corrected rendering issues with markdown-it-py plugin for ins #1493 --- funnel/utils/markdown/mdit_plugins/ins_tag.py | 67 +++++++++++++------ funnel/utils/markdown/profiles.py | 4 ++ 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/funnel/utils/markdown/mdit_plugins/ins_tag.py b/funnel/utils/markdown/mdit_plugins/ins_tag.py index a6ecef98a..32e863468 100644 --- a/funnel/utils/markdown/mdit_plugins/ins_tag.py +++ b/funnel/utils/markdown/mdit_plugins/ins_tag.py @@ -1,4 +1,8 @@ -"""Markdown-it-py plugin to introduce markup using ++inserted++.""" +""" +Markdown-it-py plugin to introduce markup using ++inserted++. + +Ported from markdown_it.rules_inline.strikethrough. +""" from typing import Any, List @@ -11,6 +15,7 @@ def ins_plugin(md: MarkdownIt): def tokenize(state: StateInline, silent: bool): + """Insert each marker as a separate text token, and add it to delimiter list.""" start = state.pos marker = state.srcCharCode[start] ch = chr(marker) @@ -28,6 +33,11 @@ def tokenize(state: StateInline, silent: bool): if length < 2: return False + if length % 2: + token = state.push("text", "", 0) + token.content = ch + length -= 1 + i = 0 while i < length: token = state.push('text', '', 0) @@ -35,8 +45,8 @@ def tokenize(state: StateInline, silent: bool): state.delimiters.append( Delimiter( marker=marker, - length=0, - jump=i // 2, + length=0, # disable "rule of 3" length checks meant for emphasis + jump=i // 2, # for `++` 1 marker = 2 characters token=len(state.tokens) - 1, end=-1, open=scanned.can_open, @@ -50,12 +60,18 @@ def tokenize(state: StateInline, silent: bool): def _post_process(state: StateInline, delimiters: List[Any]): lone_markers = [] - max_ = len(delimiters) + maximum = len(delimiters) - for i in range(0, max_): + for i in range(0, maximum): start_delim = delimiters[i] - if start_delim.marker != 0x2B or start_delim.end == -1: + if start_delim.marker != 0x2B: + i += 1 + continue + + if start_delim.end == -1: + i += 1 continue + end_delim = delimiters[start_delim.end] token = state.tokens[start_delim.token] @@ -77,7 +93,13 @@ def _post_process(state: StateInline, delimiters: List[Any]): if end_token.type == 'text' and end_token == chr(0x2B): lone_markers.append(end_delim.token - 1) - while len(lone_markers) > 0: + # If a marker sequence has an odd number of characters, it's split + # like this: `+++++` -> `+` + `++` + `++`, leaving one marker at the + # start of the sequence. + # + # So, we have to move all those markers after subsequent ins_close tags. + # + while lone_markers: i = lone_markers.pop() j = i + 1 @@ -87,24 +109,29 @@ def _post_process(state: StateInline, delimiters: List[Any]): j -= 1 if i != j: - (state.tokens[i], state.tokens[j]) = (state.tokens[j], state.tokens[i]) + token = state.tokens[j] + state.tokens[j] = state.tokens[i] + state.tokens[i] = token - md.inline.ruler.before('emphasis', 'ins', tokenize) + md.inline.ruler.before('strikethrough', 'ins', tokenize) def post_process(state: StateInline): + """Walk through delimiter list and replace text tokens with tags.""" tokens_meta = state.tokens_meta - max_ = len(state.tokens_meta) + maximum = len(state.tokens_meta) _post_process(state, state.delimiters) - for current in range(0, max_): - if ( - tokens_meta[current] is not None - and tokens_meta[current]['delimiters'] # type: ignore[index] - ): - _post_process( - state, tokens_meta[current]['delimiters'] # type: ignore[index] - ) - - md.inline.ruler2.before('emphasis', 'ins', post_process) + curr = 0 + while curr < maximum: + try: + curr_meta = tokens_meta[curr] + except IndexError: + pass + else: + if curr_meta and "delimiters" in curr_meta: + _post_process(state, curr_meta["delimiters"]) + curr += 1 + + md.inline.ruler2.before('strikethrough', 'ins', post_process) def ins_open(self, tokens, idx, options, env): return '' diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 7eb7df66d..760a66941 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -6,12 +6,15 @@ from coaster.utils import make_name +from .mdit_plugins import ins_plugin + __all__ = ['profiles', 'plugins', 'plugin_configs', 'default_markdown_options'] plugins: Dict[str, Callable] = { 'footnote': footnote.footnote_plugin, 'heading_anchors': anchors.anchors_plugin, 'tasklists': tasklists.tasklists_plugin, + 'ins': ins_plugin, } plugin_configs: Dict[str, Dict[str, Any]] = { @@ -59,6 +62,7 @@ 'footnote', 'heading_anchors', 'tasklists', + 'ins', ], }, 'text-field': { From 2628e2d29743c8a497c08f7f2bc724eec9f7ad21 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 11 Oct 2022 13:09:39 +0530 Subject: [PATCH 10/72] Cleaned up and reintroduced markdown-it-py plugin for sub & del. --- .../markdown/mdit_plugins/sub_del_tag.py | 21 ++++++++++++------- funnel/utils/markdown/profiles.py | 4 +++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/funnel/utils/markdown/mdit_plugins/sub_del_tag.py b/funnel/utils/markdown/mdit_plugins/sub_del_tag.py index 5a1e76cd9..178a81776 100644 --- a/funnel/utils/markdown/mdit_plugins/sub_del_tag.py +++ b/funnel/utils/markdown/mdit_plugins/sub_del_tag.py @@ -6,10 +6,14 @@ from markdown_it.rules_inline import StateInline from markdown_it.rules_inline.state_inline import Delimiter -__all__ = ['del_sub_plugin'] +__all__ = ['sub_del_plugin'] -def del_sub_plugin(md: MarkdownIt): +def sub_del_plugin(md: MarkdownIt): + + md.inline.ruler.disable('strikethrough') + md.inline.ruler2.disable('strikethrough') + def tokenize(state: StateInline, silent: bool): start = state.pos marker = state.srcCharCode[start] @@ -18,7 +22,7 @@ def tokenize(state: StateInline, silent: bool): if silent: return False - if marker != 0x7E: + if marker != 0x7E: # /* ~ */ return False scanned = state.scanDelims(state.pos, True) @@ -58,6 +62,11 @@ def _post_process(state: StateInline, delimiters: List[Any]): end_delim = delimiters[start_delim.end] + # If the previous delimiter has the same marker and is adjacent to this one, + # merge those into one del delimiter. + # + # `whatever` -> `whatever` + # is_del = ( i > 0 and delimiters[i - 1].end == start_delim.end + 1 @@ -89,8 +98,7 @@ def _post_process(state: StateInline, delimiters: List[Any]): i -= 1 - md.inline.ruler.disable('strikethrough') - md.inline.ruler.before('emphasis', 'sub_del', tokenize) + md.inline.ruler.before('strikethrough', 'sub_del', tokenize) def post_process(state: StateInline): _post_process(state, state.delimiters) @@ -99,8 +107,7 @@ def post_process(state: StateInline): if token and "delimiters" in token: _post_process(state, token["delimiters"]) - md.inline.ruler2.disable('strikethrough') - md.inline.ruler2.before('emphasis', 'del', post_process) + md.inline.ruler2.before('strikethrough', 'del', post_process) def del_open(self, tokens, idx, options, env): return '' diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 760a66941..2519cd680 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -6,7 +6,7 @@ from coaster.utils import make_name -from .mdit_plugins import ins_plugin +from .mdit_plugins import ins_plugin, sub_del_plugin __all__ = ['profiles', 'plugins', 'plugin_configs', 'default_markdown_options'] @@ -15,6 +15,7 @@ 'heading_anchors': anchors.anchors_plugin, 'tasklists': tasklists.tasklists_plugin, 'ins': ins_plugin, + 'sub_del': sub_del_plugin, } plugin_configs: Dict[str, Dict[str, Any]] = { @@ -63,6 +64,7 @@ 'heading_anchors', 'tasklists', 'ins', + 'sub_del', ], }, 'text-field': { From 4c61e7fde2ef9700345ba2fff99cda8360e4103b Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 14 Oct 2022 15:03:39 +0530 Subject: [PATCH 11/72] Ported markdown-it-sup #1493 --- funnel/utils/markdown/mdit_plugins/sup_tag.py | 133 +++++++----------- funnel/utils/markdown/profiles.py | 4 +- 2 files changed, 53 insertions(+), 84 deletions(-) diff --git a/funnel/utils/markdown/mdit_plugins/sup_tag.py b/funnel/utils/markdown/mdit_plugins/sup_tag.py index f64bd561e..9d71f9ff3 100644 --- a/funnel/utils/markdown/mdit_plugins/sup_tag.py +++ b/funnel/utils/markdown/mdit_plugins/sup_tag.py @@ -1,10 +1,14 @@ -"""Markdown-it-py plugin to introduce markup using ^superscript^.""" +""" +Markdown-it-py plugin to introduce markup using ^superscript^. -from typing import Any, List +Ported from +https://github.com/markdown-it/markdown-it-sup/blob/master/dist/markdown-it-sup.js +""" + +import re from markdown_it import MarkdownIt from markdown_it.rules_inline import StateInline -from markdown_it.rules_inline.state_inline import Delimiter __all__ = ['sup_plugin'] @@ -13,7 +17,8 @@ def sup_plugin(md: MarkdownIt): def tokenize(state: StateInline, silent: bool): start = state.pos marker = state.srcCharCode[start] - ch = chr(marker) + maximum = state.posMax + found = False if silent: return False @@ -21,87 +26,49 @@ def tokenize(state: StateInline, silent: bool): if marker != 0x5E: return False - scanned = state.scanDelims(state.pos, True) - - length = scanned.length - - i = 0 - while i < length: - token = state.push('text', '', 0) - token.content = ch - state.delimiters.append( - Delimiter( - marker=marker, - length=0, - jump=i, - token=len(state.tokens) - 1, - end=-1, - open=scanned.can_open, - close=scanned.can_close, - ) - ) - i += 1 - - state.pos += scanned.length + # Don't run any pairs in validation mode + if start + 2 >= maximum: + return False + + state.pos = start + 1 + + while state.pos < maximum: + if state.srcCharCode[state.pos] == 0x5E: + found = True + break + state.md.inline.skipToken(state) + + if not found or start + 1 == state.pos: + state.pos = start + return False + + content = state.src[start + 1 : state.pos] + + # Don't allow unescaped spaces/newlines inside + if re.search(r'(^|[^\\])(\\\\)*\s', content) is not None: + state.pos = start + return False + + state.posMax = state.pos + state.pos = start + 1 + + # Earlier we checked "not silent", but this implementation does not need it + token = state.push('sup_open', 'sup', 1) + token.markup = '^' + + token = state.push('text', '', 0) + token.content = content.replace( + r'\\([ \\!"#$%&\'()*+,.\/:;<=>?@[\]^_`{|}~-])', '$1' + ) + + token = state.push('sup_close', 'sup', -1) + token.markup = '^' + + state.pos = state.posMax + 1 + state.posMax = maximum return True - def _post_process(state: StateInline, delimiters: List[Any]): - lone_markers = [] - max_ = len(delimiters) - - for i in range(0, max_): - start_delim = delimiters[i] - if start_delim.marker != 0x5E or start_delim.end == -1: - continue - end_delim = delimiters[start_delim.end] - - token = state.tokens[start_delim.token] - token.type = 'sup_open' - token.tag = 'sup' - token.nesting = 1 - token.markup = '^' - token.content = '' - - token = state.tokens[end_delim.token] - token.type = 'sup_close' - token.tag = 'sup' - token.nesting = -1 - token.markup = '^' - token.content = '' - - end_token = state.tokens[end_delim.token - 1] - - if end_token.type == 'text' and end_token == chr(0x5E): - lone_markers.append(end_delim.token - 1) - - while len(lone_markers) > 0: - i = lone_markers.pop() - j = i + 1 - - while j < len(state.tokens) and state.tokens[j].type == 'sup_close': - j += 1 - - j -= 1 - - if i != j: - (state.tokens[i], state.tokens[j]) = (state.tokens[j], state.tokens[i]) - - md.inline.ruler.before('emphasis', 'sup', tokenize) - - def post_process(state: StateInline): - tokens_meta = state.tokens_meta - max_ = len(state.tokens_meta) - _post_process(state, state.delimiters) - for current in range(0, max_): - if ( - tokens_meta[current] is not None - and tokens_meta[current]['delimiters'] # type: ignore[index] - ): - _post_process( - state, tokens_meta[current]['delimiters'] # type: ignore[index] - ) - - md.inline.ruler2.before('emphasis', 'sup', post_process) + md.inline.ruler.before('strikethrough', 'sup', tokenize) def sup_open(self, tokens, idx, options, env): return '' diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 2519cd680..e8ad21a94 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -6,7 +6,7 @@ from coaster.utils import make_name -from .mdit_plugins import ins_plugin, sub_del_plugin +from .mdit_plugins import ins_plugin, sub_del_plugin, sup_plugin __all__ = ['profiles', 'plugins', 'plugin_configs', 'default_markdown_options'] @@ -16,6 +16,7 @@ 'tasklists': tasklists.tasklists_plugin, 'ins': ins_plugin, 'sub_del': sub_del_plugin, + 'sup': sup_plugin, } plugin_configs: Dict[str, Dict[str, Any]] = { @@ -65,6 +66,7 @@ 'tasklists', 'ins', 'sub_del', + 'sup', ], }, 'text-field': { From 6708e79b08daf187155ed5e52861ebdad1e31cb7 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 17 Oct 2022 03:32:36 +0530 Subject: [PATCH 12/72] Markdown-it-py sub tag plugin now ported from javascript. Correction in rule placement for sup tag plugin. Simple plugin to replace with . #1493 --- .../utils/markdown/mdit_plugins/__init__.py | 3 +- funnel/utils/markdown/mdit_plugins/del_tag.py | 16 +++ .../markdown/mdit_plugins/sub_del_tag.py | 127 ------------------ funnel/utils/markdown/mdit_plugins/sub_tag.py | 80 +++++++++++ funnel/utils/markdown/mdit_plugins/sup_tag.py | 2 +- funnel/utils/markdown/profiles.py | 8 +- 6 files changed, 104 insertions(+), 132 deletions(-) create mode 100644 funnel/utils/markdown/mdit_plugins/del_tag.py delete mode 100644 funnel/utils/markdown/mdit_plugins/sub_del_tag.py create mode 100644 funnel/utils/markdown/mdit_plugins/sub_tag.py diff --git a/funnel/utils/markdown/mdit_plugins/__init__.py b/funnel/utils/markdown/mdit_plugins/__init__.py index 47b1aef7d..5e4d11fe1 100644 --- a/funnel/utils/markdown/mdit_plugins/__init__.py +++ b/funnel/utils/markdown/mdit_plugins/__init__.py @@ -1,6 +1,7 @@ """Plugins for markdown-it-py.""" # flake8: noqa +from .del_tag import * from .ins_tag import * -from .sub_del_tag import * +from .sub_tag import * from .sup_tag import * diff --git a/funnel/utils/markdown/mdit_plugins/del_tag.py b/funnel/utils/markdown/mdit_plugins/del_tag.py new file mode 100644 index 000000000..9d7902f5a --- /dev/null +++ b/funnel/utils/markdown/mdit_plugins/del_tag.py @@ -0,0 +1,16 @@ +"""Markdown-it-py plugin to replace with for ~~.""" + +from markdown_it import MarkdownIt + +__all__ = ['del_plugin'] + + +def del_plugin(md: MarkdownIt): + def del_open(self, tokens, idx, options, env): + return '' + + def del_close(self, tokens, idx, options, env): + return '' + + md.add_render_rule('s_open', del_open) + md.add_render_rule('s_close', del_close) diff --git a/funnel/utils/markdown/mdit_plugins/sub_del_tag.py b/funnel/utils/markdown/mdit_plugins/sub_del_tag.py deleted file mode 100644 index 178a81776..000000000 --- a/funnel/utils/markdown/mdit_plugins/sub_del_tag.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Markdown-it-py plugin for & using ~subscricpt~ & ~~deleted~~.""" - -from typing import Any, List - -from markdown_it import MarkdownIt -from markdown_it.rules_inline import StateInline -from markdown_it.rules_inline.state_inline import Delimiter - -__all__ = ['sub_del_plugin'] - - -def sub_del_plugin(md: MarkdownIt): - - md.inline.ruler.disable('strikethrough') - md.inline.ruler2.disable('strikethrough') - - def tokenize(state: StateInline, silent: bool): - start = state.pos - marker = state.srcCharCode[start] - chr(marker) - - if silent: - return False - - if marker != 0x7E: # /* ~ */ - return False - - scanned = state.scanDelims(state.pos, True) - - for i in range(scanned.length): - token = state.push("text", "", 0) - token.content = chr(marker) - state.delimiters.append( - Delimiter( - marker=marker, - length=scanned.length, - jump=i, - token=len(state.tokens) - 1, - end=-1, - open=scanned.can_open, - close=scanned.can_close, - ) - ) - - state.pos += scanned.length - return True - - def _post_process(state: StateInline, delimiters: List[Any]): - i = len(delimiters) - 1 - while i >= 0: - start_delim = delimiters[i] - - # /* ~ */ - if start_delim.marker != 0x7E: - i -= 1 - continue - - # Process only opening markers - if start_delim.end == -1: - i -= 1 - continue - - end_delim = delimiters[start_delim.end] - - # If the previous delimiter has the same marker and is adjacent to this one, - # merge those into one del delimiter. - # - # `whatever` -> `whatever` - # - is_del = ( - i > 0 - and delimiters[i - 1].end == start_delim.end + 1 - and delimiters[i - 1].token == start_delim.token - 1 - and delimiters[start_delim.end + 1].token == end_delim.token + 1 - and delimiters[i - 1].marker == start_delim.marker - ) - - ch = chr(start_delim.marker) - - token = state.tokens[start_delim.token] - token.type = "del_open" if is_del else "sub_open" - token.tag = "del" if is_del else "sub" - token.nesting = 1 - token.markup = ch + ch if is_del else ch - token.content = "" - - token = state.tokens[end_delim.token] - token.type = "del_close" if is_del else "sub_close" - token.tag = "del" if is_del else "sub" - token.nesting = -1 - token.markup = ch + ch if is_del else ch - token.content = "" - - if is_del: - state.tokens[delimiters[i - 1].token].content = "" - state.tokens[delimiters[start_delim.end + 1].token].content = "" - i -= 1 - - i -= 1 - - md.inline.ruler.before('strikethrough', 'sub_del', tokenize) - - def post_process(state: StateInline): - _post_process(state, state.delimiters) - - for token in state.tokens_meta: - if token and "delimiters" in token: - _post_process(state, token["delimiters"]) - - md.inline.ruler2.before('strikethrough', 'del', post_process) - - def del_open(self, tokens, idx, options, env): - return '' - - def del_close(self, tokens, idx, options, env): - return '' - - def sub_open(self, tokens, idx, options, env): - return '' - - def sub_close(self, tokens, idx, options, env): - return '' - - md.add_render_rule('del_open', del_open) - md.add_render_rule('del_close', del_close) - md.add_render_rule('sub_open', sub_open) - md.add_render_rule('sub_close', sub_close) diff --git a/funnel/utils/markdown/mdit_plugins/sub_tag.py b/funnel/utils/markdown/mdit_plugins/sub_tag.py new file mode 100644 index 000000000..4c4a2f9cc --- /dev/null +++ b/funnel/utils/markdown/mdit_plugins/sub_tag.py @@ -0,0 +1,80 @@ +""" +Markdown-it-py plugin to introduce markup using ~subscript~. + +Ported from +https://github.com/markdown-it/markdown-it-sub/blob/master/dist/markdown-it-sub.js +""" + +import re + +from markdown_it import MarkdownIt +from markdown_it.rules_inline import StateInline + +__all__ = ['sub_plugin'] + + +def sub_plugin(md: MarkdownIt): + def tokenize(state: StateInline, silent: bool): + start = state.pos + marker = state.srcCharCode[start] + maximum = state.posMax + found = False + + if silent: + return False + + if marker != 0x7E: + return False + + # Don't run any pairs in validation mode + if start + 2 >= maximum: + return False + + state.pos = start + 1 + + while state.pos < maximum: + if state.srcCharCode[state.pos] == 0x7E: + found = True + break + state.md.inline.skipToken(state) + + if not found or start + 1 == state.pos: + state.pos = start + return False + + content = state.src[start + 1 : state.pos] + + # Don't allow unescaped spaces/newlines inside + if re.search(r'(^|[^\\])(\\\\)*\s', content) is not None: + state.pos = start + return False + + state.posMax = state.pos + state.pos = start + 1 + + # Earlier we checked "not silent", but this implementation does not need it + token = state.push('sub_open', 'sub', 1) + token.markup = '~' + + token = state.push('text', '', 0) + token.content = content.replace( + r'\\([ \\!"#$%&\'()*+,.\/:;<=>?@[\]^_`{|}~-])', '$1' + ) + + token = state.push('sub_close', 'sub', -1) + token.markup = '~' + + state.pos = state.posMax + 1 + state.posMax = maximum + return True + + md.inline.ruler.after('emphasis', 'sub', tokenize) + + def sub_open(self, tokens, idx, options, env): + return '' + + def sub_close(self, tokens, idx, options, env): + return '' + + md.add_render_rule('sub_open', sub_open) + md.add_render_rule('sub_close', sub_close) diff --git a/funnel/utils/markdown/mdit_plugins/sup_tag.py b/funnel/utils/markdown/mdit_plugins/sup_tag.py index 9d71f9ff3..47dc19a20 100644 --- a/funnel/utils/markdown/mdit_plugins/sup_tag.py +++ b/funnel/utils/markdown/mdit_plugins/sup_tag.py @@ -68,7 +68,7 @@ def tokenize(state: StateInline, silent: bool): state.posMax = maximum return True - md.inline.ruler.before('strikethrough', 'sup', tokenize) + md.inline.ruler.after('emphasis', 'sup', tokenize) def sup_open(self, tokens, idx, options, env): return '' diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index e8ad21a94..88a1e8c8a 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -6,7 +6,7 @@ from coaster.utils import make_name -from .mdit_plugins import ins_plugin, sub_del_plugin, sup_plugin +from .mdit_plugins import del_plugin, ins_plugin, sub_plugin, sup_plugin __all__ = ['profiles', 'plugins', 'plugin_configs', 'default_markdown_options'] @@ -15,7 +15,8 @@ 'heading_anchors': anchors.anchors_plugin, 'tasklists': tasklists.tasklists_plugin, 'ins': ins_plugin, - 'sub_del': sub_del_plugin, + 'del': del_plugin, + 'sub': sub_plugin, 'sup': sup_plugin, } @@ -65,7 +66,8 @@ 'heading_anchors', 'tasklists', 'ins', - 'sub_del', + 'del', + 'sub', 'sup', ], }, From db0c476f608bb7fbc9812a6c379cba21b2973f33 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 17 Oct 2022 11:41:48 +0530 Subject: [PATCH 13/72] Plugin for mark tag using == #1493 --- .../utils/markdown/mdit_plugins/__init__.py | 1 + .../utils/markdown/mdit_plugins/mark_tag.py | 143 ++++++++++++++++++ funnel/utils/markdown/profiles.py | 4 +- 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 funnel/utils/markdown/mdit_plugins/mark_tag.py diff --git a/funnel/utils/markdown/mdit_plugins/__init__.py b/funnel/utils/markdown/mdit_plugins/__init__.py index 5e4d11fe1..409da9083 100644 --- a/funnel/utils/markdown/mdit_plugins/__init__.py +++ b/funnel/utils/markdown/mdit_plugins/__init__.py @@ -3,5 +3,6 @@ from .del_tag import * from .ins_tag import * +from .mark_tag import * from .sub_tag import * from .sup_tag import * diff --git a/funnel/utils/markdown/mdit_plugins/mark_tag.py b/funnel/utils/markdown/mdit_plugins/mark_tag.py new file mode 100644 index 000000000..e92c3ee14 --- /dev/null +++ b/funnel/utils/markdown/mdit_plugins/mark_tag.py @@ -0,0 +1,143 @@ +""" +Markdown-it-py plugin to introduce markup using ==marked==. + +Ported from markdown_it.rules_inline.strikethrough. +""" + +from typing import Any, List + +from markdown_it import MarkdownIt +from markdown_it.rules_inline import StateInline +from markdown_it.rules_inline.state_inline import Delimiter + +__all__ = ['mark_plugin'] + + +def mark_plugin(md: MarkdownIt): + def tokenize(state: StateInline, silent: bool): + """Insert each marker as a separate text token, and add it to delimiter list.""" + start = state.pos + marker = state.srcCharCode[start] + ch = chr(marker) + + if silent: + return False + + if marker != 0x3D: + return False + + scanned = state.scanDelims(state.pos, True) + + length = scanned.length + + if length < 2: + return False + + if length % 2: + token = state.push("text", "", 0) + token.content = ch + length -= 1 + + i = 0 + while i < length: + token = state.push('text', '', 0) + token.content = ch + ch + state.delimiters.append( + Delimiter( + marker=marker, + length=0, # disable "rule of 3" length checks meant for emphasis + jump=i // 2, # for `==` 1 marker = 2 characters + token=len(state.tokens) - 1, + end=-1, + open=scanned.can_open, + close=scanned.can_close, + ) + ) + i += 2 + + state.pos += scanned.length + return True + + def _post_process(state: StateInline, delimiters: List[Any]): + lone_markers = [] + maximum = len(delimiters) + + for i in range(0, maximum): + start_delim = delimiters[i] + if start_delim.marker != 0x3D: + i += 1 + continue + + if start_delim.end == -1: + i += 1 + continue + + end_delim = delimiters[start_delim.end] + + token = state.tokens[start_delim.token] + token.type = 'mark_open' + token.tag = 'mark' + token.nesting = 1 + token.markup = '==' + token.content = '' + + token = state.tokens[end_delim.token] + token.type = 'mark_close' + token.tag = 'mark' + token.nesting = -1 + token.markup = '==' + token.content = '' + + end_token = state.tokens[end_delim.token - 1] + + if end_token.type == 'text' and end_token == chr(0x3D): + lone_markers.append(end_delim.token - 1) + + # If a marker sequence has an odd number of characters, it's split + # like this: `=====` -> `=` + `==` + `==`, leaving one marker at the + # start of the sequence. + # + # So, we have to move all those markers after subsequent mark_close tags. + # + while lone_markers: + i = lone_markers.pop() + j = i + 1 + + while j < len(state.tokens) and state.tokens[j].type == 'mark_close': + j += 1 + + j -= 1 + + if i != j: + token = state.tokens[j] + state.tokens[j] = state.tokens[i] + state.tokens[i] = token + + md.inline.ruler.before('strikethrough', 'mark', tokenize) + + def post_process(state: StateInline): + """Walk through delimiter list and replace text tokens with tags.""" + tokens_meta = state.tokens_meta + maximum = len(state.tokens_meta) + _post_process(state, state.delimiters) + curr = 0 + while curr < maximum: + try: + curr_meta = tokens_meta[curr] + except IndexError: + pass + else: + if curr_meta and "delimiters" in curr_meta: + _post_process(state, curr_meta["delimiters"]) + curr += 1 + + md.inline.ruler2.before('strikethrough', 'mark', post_process) + + def mark_open(self, tokens, idx, options, env): + return '' + + def mark_close(self, tokens, idx, options, env): + return '' + + md.add_render_rule('mark_open', mark_open) + md.add_render_rule('mark_close', mark_close) diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 88a1e8c8a..441f5e932 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -6,7 +6,7 @@ from coaster.utils import make_name -from .mdit_plugins import del_plugin, ins_plugin, sub_plugin, sup_plugin +from .mdit_plugins import del_plugin, ins_plugin, mark_plugin, sub_plugin, sup_plugin __all__ = ['profiles', 'plugins', 'plugin_configs', 'default_markdown_options'] @@ -18,6 +18,7 @@ 'del': del_plugin, 'sub': sub_plugin, 'sup': sup_plugin, + 'mark': mark_plugin, } plugin_configs: Dict[str, Dict[str, Any]] = { @@ -69,6 +70,7 @@ 'del', 'sub', 'sup', + 'mark', ], }, 'text-field': { From 522f1144a054dfab9201c6dc783beda07c0270c9 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 19 Oct 2022 21:32:08 +0530 Subject: [PATCH 14/72] Markdown-it-py plugin for embeds, ported from mdit_py_plugins.container & mdit_py_plugins.colon_fence. #1496 --- .../utils/markdown/mdit_plugins/__init__.py | 1 + funnel/utils/markdown/mdit_plugins/embeds.py | 147 ++++++++++++++++++ funnel/utils/markdown/profiles.py | 5 + 3 files changed, 153 insertions(+) create mode 100644 funnel/utils/markdown/mdit_plugins/embeds.py diff --git a/funnel/utils/markdown/mdit_plugins/__init__.py b/funnel/utils/markdown/mdit_plugins/__init__.py index 47b1aef7d..be74e310f 100644 --- a/funnel/utils/markdown/mdit_plugins/__init__.py +++ b/funnel/utils/markdown/mdit_plugins/__init__.py @@ -1,6 +1,7 @@ """Plugins for markdown-it-py.""" # flake8: noqa +from .embeds import * from .ins_tag import * from .sub_del_tag import * from .sup_tag import * diff --git a/funnel/utils/markdown/mdit_plugins/embeds.py b/funnel/utils/markdown/mdit_plugins/embeds.py new file mode 100644 index 000000000..fdee79ac5 --- /dev/null +++ b/funnel/utils/markdown/mdit_plugins/embeds.py @@ -0,0 +1,147 @@ +""" +Markdown-it-py plugin to handle embeds. + +Ported from mdit_py_plugins.container +and mdit_py_plugins.colon_fence. +""" + +from math import floor +import re + +from markdown_it import MarkdownIt +from markdown_it.common.utils import charCodeAt +from markdown_it.rules_block import StateBlock + + +def embeds_plugin( + md: MarkdownIt, + name: str, + marker: str = '`', +): + def validate(params: str, *args): + results = re.findall(r'^{\s*([a-zA-Z0-9_\-]+)\s*}.*$', params.strip()) + return len(results) != 0 and results[0] == name + + def render(self, tokens, idx, _options, env): + token = tokens[idx] + content = md.utils.escapeHtml(token.content) + + return ( + '
' + + content + + '
\n' + ) + + min_markers = 3 + marker_str = marker + marker_char = charCodeAt(marker_str, 0) + marker_len = len(marker_str) + + def embeds_func(state: StateBlock, start_line: int, end_line: int, silent: bool): + + auto_closed = False + start = state.bMarks[start_line] + state.tShift[start_line] + maximum = state.eMarks[start_line] + + # Check out the first character quickly, + # this should filter out most of non-containers + if marker_char != state.srcCharCode[start]: + return False + + # Check out the rest of the marker string + pos = start + 1 + while pos <= maximum: + try: + character = state.src[pos] + except IndexError: + break + if marker_str[(pos - start) % marker_len] != character: + break + pos += 1 + + marker_count = floor((pos - start) / marker_len) + if marker_count < min_markers: + return False + pos -= (pos - start) % marker_len + + markup = state.src[start:pos] + params = state.src[pos:maximum] + + if not validate(params, markup): + return False + + # Since start is found, we can report success here in validation mode + if silent: + return True + + # Search for the end of the block + next_line = start_line + + while True: + next_line += 1 + if next_line >= end_line: + # unclosed block should be autoclosed by end of document. + # also block seems to be autoclosed by end of parent + break + + start = state.bMarks[next_line] + state.tShift[next_line] + maximum = state.eMarks[next_line] + + if start < maximum and state.sCount[next_line] < state.blkIndent: + # non-empty line with negative indent should stop the list: + # - ``` + # test + break + + if marker_char != state.srcCharCode[start]: + continue + + if state.sCount[next_line] - state.blkIndent >= 4: + # closing fence should be indented less than 4 spaces + continue + + pos = start + 1 + while pos <= maximum: + try: + character = state.src[pos] + except IndexError: + break + if marker_str[(pos - start) % marker_len] != character: + break + pos += 1 + + # closing code fence must be at least as long as the opening one + if floor((pos - start) / marker_len) < marker_count: + continue + + # make sure tail has spaces only + pos -= (pos - start) % marker_len + pos = state.skipSpaces(pos) + + if pos < maximum: + continue + + # found! + auto_closed = True + break + + state.line = next_line + (1 if auto_closed else 0) + + # Borrowed from mdit_py_plugins.colon_fence + token = state.push(f'embed_{name}', 'div', 0) + token.info = params + token.content = state.getLines(start_line + 1, next_line, marker_count, True) + token.markup = markup + token.map = [start_line, state.line] + + return True + + md.block.ruler.before( + 'fence', + f'embed_{name}', + embeds_func, + ) + + md.add_render_rule(f'embed_{name}', render) diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 7eb7df66d..f241690c5 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -6,12 +6,15 @@ from coaster.utils import make_name +from .mdit_plugins import embeds_plugin + __all__ = ['profiles', 'plugins', 'plugin_configs', 'default_markdown_options'] plugins: Dict[str, Callable] = { 'footnote': footnote.footnote_plugin, 'heading_anchors': anchors.anchors_plugin, 'tasklists': tasklists.tasklists_plugin, + 'markmap': embeds_plugin, } plugin_configs: Dict[str, Dict[str, Any]] = { @@ -22,6 +25,7 @@ 'permalink': True, }, 'tasklists': {'enabled': True, 'label': True, 'label_after': False}, + 'markmap': {'name': 'markmap'}, } default_markdown_options = { @@ -59,6 +63,7 @@ 'footnote', 'heading_anchors', 'tasklists', + 'markmap', ], }, 'text-field': { From 6f0c9da14529db9d24359c525066e89f985a0af3 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 20 Oct 2022 11:46:21 +0530 Subject: [PATCH 15/72] Small correction to 522f114 #1496 --- funnel/utils/markdown/mdit_plugins/embeds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funnel/utils/markdown/mdit_plugins/embeds.py b/funnel/utils/markdown/mdit_plugins/embeds.py index fdee79ac5..21ce2351c 100644 --- a/funnel/utils/markdown/mdit_plugins/embeds.py +++ b/funnel/utils/markdown/mdit_plugins/embeds.py @@ -31,7 +31,7 @@ def render(self, tokens, idx, _options, env): + (f' class="md-embed-{name}" ') + '>
' + content - + '
\n' + + '
\n' ) min_markers = 3 From 289dda26d1df49fe59326be3f7541c2eae508df6 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 27 Oct 2022 18:31:27 +0530 Subject: [PATCH 16/72] Markmap, Mermaid and Vega integration handled with new Markdown #1496 --- funnel/assets/js/utils/markmap.js | 10 ++- funnel/assets/js/utils/mermaid.js | 27 +++++-- funnel/assets/js/utils/vegaembed.js | 28 ++++--- funnel/assets/sass/_layout.scss | 77 ++++++++++++-------- funnel/utils/markdown/mdit_plugins/embeds.py | 2 +- funnel/utils/markdown/profiles.py | 6 ++ 6 files changed, 99 insertions(+), 51 deletions(-) diff --git a/funnel/assets/js/utils/markmap.js b/funnel/assets/js/utils/markmap.js index b4e8784f6..005931b62 100644 --- a/funnel/assets/js/utils/markmap.js +++ b/funnel/assets/js/utils/markmap.js @@ -1,10 +1,12 @@ const MarkmapEmbed = { addMarkmap() { - $('.language-markmap').each(function embedMarkmap() { - $(this).addClass('embed-added'); - $(this).find('code').addClass('markmap'); + $('.md-embed-markmap').each(function embedMarkmap() { + $(this).find('.embed-content').addClass('markmap'); }); window.markmap.autoLoader.renderAll(); + $('.md-embed-markmap').each(function embedMarkmap() { + $(this).addClass('activated'); + }); }, loadMarkmap() { const self = this; @@ -52,7 +54,7 @@ const MarkmapEmbed = { }, init(containerDiv) { this.containerDiv = containerDiv; - if ($('.language-markmap').length > 0) { + if ($('.md-embed-markmap:not(.activated)').length > 0) { this.loadMarkmap(); } }, diff --git a/funnel/assets/js/utils/mermaid.js b/funnel/assets/js/utils/mermaid.js index 01afc7cae..65db6473a 100644 --- a/funnel/assets/js/utils/mermaid.js +++ b/funnel/assets/js/utils/mermaid.js @@ -1,10 +1,26 @@ const MermaidEmbed = { addMermaid() { - $('.language-mermaid').each(function embedMarkmap() { - $(this).addClass('embed-added'); - $(this).find('code').addClass('mermaid'); + const instances = $('.md-embed-mermaid:not(.activating):not(.activated)'); + let idCount = $('.md-embed-mermaid.activating, .md-embed-mermaid.activated').length; + const idMarker = 'mermaid_elem_'; + instances.each(function embedMarkmap() { + const root = $(this); + root.addClass('activating'); + const elem = root.find('.embed-content'); + const definition = elem.text(); + let elemId = elem.attr('id'); + if (!elemId) { + elemId = `${idMarker}${idCount}`; + do { + idCount += 1; + } while ($(`#${idMarker}${idCount}`).length > 0); + } + window.mermaid.render(elemId, definition, (svg) => { + elem.html(svg); + root.addClass('activated'); + root.removeClass('activating'); + }); }); - window.mermaid.initialize({ startOnLoad: true }); }, loadMermaid() { const self = this; @@ -14,6 +30,7 @@ const MermaidEmbed = { dataType: 'script', cache: true, }).done(() => { + window.mermaid.initialize({ startOnLoad: false }); self.addMermaid(); }); } else { @@ -21,7 +38,7 @@ const MermaidEmbed = { } }, init() { - if ($('.language-mermaid').length > 0) { + if ($('.md-embed-mermaid:not(.activated)').length > 0) { this.loadMermaid(); } }, diff --git a/funnel/assets/js/utils/vegaembed.js b/funnel/assets/js/utils/vegaembed.js index 5ceb2bf59..7daa75b40 100644 --- a/funnel/assets/js/utils/vegaembed.js +++ b/funnel/assets/js/utils/vegaembed.js @@ -1,20 +1,28 @@ /* global vegaEmbed */ function addVegaChart() { - $('.language-vega-lite').each(function embedVegaChart() { - vegaEmbed(this, JSON.parse($(this).find('code').text()), { - renderer: 'svg', - actions: { - source: false, - editor: false, - compiled: false, - }, + $('.md-embed-vega-lite:not(.activated)').each(async function embedVegaChart() { + const root = $(this); + const embedded = await vegaEmbed( + this, + JSON.parse($(this).find('.embed-content').text()), + { + renderer: 'svg', + actions: { + source: false, + editor: false, + compiled: false, + }, + } + ); + embedded.view.runAfter(() => { + root.addClass('activated'); }); }); } function addVegaSupport() { - if ($('.language-vega-lite').length > 0) { + if ($('.md-embed-vega-lite:not(.activated)').length > 0) { const vegaliteCDN = [ 'https://cdn.jsdelivr.net/npm/vega@5', 'https://cdn.jsdelivr.net/npm/vega-lite@5', @@ -32,7 +40,7 @@ function addVegaSupport() { vegaliteUrl += 1; loadVegaScript(); } - // Once all vega js is loaded, initialize vega visualization on all pre tags with class 'language-vega-lite' + // Once all vega js is loaded, initialize vega visualization on all pre tags with class 'md-embed-vega-lite' if (vegaliteUrl === vegaliteCDN.length) { addVegaChart(); } diff --git a/funnel/assets/sass/_layout.scss b/funnel/assets/sass/_layout.scss index 4f97907c5..df59759e7 100644 --- a/funnel/assets/sass/_layout.scss +++ b/funnel/assets/sass/_layout.scss @@ -860,44 +860,59 @@ cursor: pointer; } -.language-placeholder { - display: none; -} - -.language-placeholder.embed-added { - display: block; - &.language-markmap { - margin-bottom: 0px; - } +.md-embed { + visibility: hidden; + border: 1px solid black; + background: linear-gradient(45deg, #bbb, #eee); canvas, svg { width: 100% !important; - height: 80vh; - border: 1px solid black; - background: linear-gradient(45deg, #bbb, #eee); + max-width: 100% !important; + background-color: transparent !important; } - canvas { - min-height: 150px; + .embed-content { + white-space: pre; } -} - -@media (max-aspect-ratio: 1/1) { - .language-placeholder.embed-added svg { - height: 40vh; + &.activated { + visibility: visible; } -} - -@media (max-aspect-ratio: 1/2) { - .language-placeholder.embed-added svg { - height: 30vh; + &.md-embed-markmap { + margin-bottom: 0px; + .embed-content { + width: 100%; + aspect-ratio: 4 / 3; + svg { + height: 100%; + } + } } -} - -.language-placeholder.language-vega-lite.vega-embed { - display: block; - canvas, - svg { - max-width: 100% !important; + &.md-embed-vega-lite.vega-embed { + padding: 2.5%; + display: block !important; + svg.marks { + height: auto; + } + summary { + top: 2px; + right: 15px; + } + .vega-actions { + right: 6px; + top: 37px; + } + &.has-actions { + padding-top: 40px; + } + } + &.md-embed-mermaid { + text-align: left; + &.activated { + text-align: center; + } + } + &.md-embed-vega-lite.vega-embed, + &.md-embed-mermaid { + padding: 2.5%; } } diff --git a/funnel/utils/markdown/mdit_plugins/embeds.py b/funnel/utils/markdown/mdit_plugins/embeds.py index 21ce2351c..70f867c3d 100644 --- a/funnel/utils/markdown/mdit_plugins/embeds.py +++ b/funnel/utils/markdown/mdit_plugins/embeds.py @@ -28,7 +28,7 @@ def render(self, tokens, idx, _options, env): return ( '
' + content + '
\n' diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index f241690c5..7aa166f90 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -15,6 +15,8 @@ 'heading_anchors': anchors.anchors_plugin, 'tasklists': tasklists.tasklists_plugin, 'markmap': embeds_plugin, + 'vega-lite': embeds_plugin, + 'mermaid': embeds_plugin, } plugin_configs: Dict[str, Dict[str, Any]] = { @@ -26,6 +28,8 @@ }, 'tasklists': {'enabled': True, 'label': True, 'label_after': False}, 'markmap': {'name': 'markmap'}, + 'vega-lite': {'name': 'vega-lite'}, + 'mermaid': {'name': 'mermaid'}, } default_markdown_options = { @@ -64,6 +68,8 @@ 'heading_anchors', 'tasklists', 'markmap', + 'vega-lite', + 'mermaid', ], }, 'text-field': { From 499566eb21497a7374101b615fef5ce4a873815b Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 27 Oct 2022 19:39:59 +0530 Subject: [PATCH 17/72] Resize markmap container on window resize #1496 --- funnel/assets/js/utils/markmap.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/funnel/assets/js/utils/markmap.js b/funnel/assets/js/utils/markmap.js index 005931b62..addc79c10 100644 --- a/funnel/assets/js/utils/markmap.js +++ b/funnel/assets/js/utils/markmap.js @@ -8,6 +8,18 @@ const MarkmapEmbed = { $(this).addClass('activated'); }); }, + resizeTimer: null, + resizeMarkmapContainers() { + if (this.resizeTimer) clearTimeout(this.resizeTimer); + this.resizeTimer = setTimeout(() => { + $('.md-embed-markmap.activated svg').each(function mmresized() { + const circles = $(this).find('circle'); + const firstNode = circles[circles.length - 1]; + firstNode.dispatchEvent(new Event('click')); + firstNode.dispatchEvent(new Event('click')); + }); + }, 500); + }, loadMarkmap() { const self = this; const CDN = [ @@ -43,6 +55,7 @@ const MarkmapEmbed = { loadMarkmapScript(); } else { self.addMarkmap(); + window.addEventListener('resize', this.resizeMarkmapContainers); } }); }; From 450e31c0cc4a504bc42f602e850286bb4636896e Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 27 Oct 2022 20:09:53 +0530 Subject: [PATCH 18/72] Use document profile for Proposal.body #1485 --- funnel/models/proposal.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/funnel/models/proposal.py b/funnel/models/proposal.py index 146f6cff8..fa91a4359 100644 --- a/funnel/models/proposal.py +++ b/funnel/models/proposal.py @@ -12,19 +12,14 @@ BaseMixin, BaseScopedIdNameMixin, Mapped, - MarkdownColumn, + MarkdownColumnNative, TSVectorType, UuidMixin, db, sa, ) from .comment import SET_TYPE, Commentset -from .helpers import ( - add_search_trigger, - markdown_content_options, - reopen, - visual_field_delimiter, -) +from .helpers import add_search_trigger, reopen, visual_field_delimiter from .project import Project from .project_membership import project_child_role_map from .reorder_mixin import ReorderMixin @@ -177,9 +172,7 @@ class Proposal( # type: ignore[misc] back_populates='proposal', ) - body = MarkdownColumn( - 'body', nullable=False, default='', options=markdown_content_options - ) + body = MarkdownColumnNative('body', profile='document', nullable=False, default='') description = sa.Column(sa.Unicode, nullable=False, default='') custom_description = sa.Column(sa.Boolean, nullable=False, default=False) template = sa.Column(sa.Boolean, nullable=False, default=False) From 40d6b1b2369c3a3de6bf1fdf87d548fc156d5fb6 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 28 Oct 2022 13:10:07 +0530 Subject: [PATCH 19/72] Resolves some of the issues in review for #1480 --- funnel/models/comment.py | 4 ++-- funnel/models/helpers.py | 13 +++++-------- funnel/models/update.py | 4 ++-- funnel/models/venue.py | 6 +++--- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/funnel/models/comment.py b/funnel/models/comment.py index f38dacca4..a3665fc2a 100644 --- a/funnel/models/comment.py +++ b/funnel/models/comment.py @@ -16,11 +16,11 @@ from . import ( BaseMixin, Mapped, - MarkdownColumnNative, TSVectorType, UuidMixin, db, hybrid_property, + markdown_cached_column, sa, ) from .helpers import MessageComposite, add_search_trigger, reopen @@ -220,7 +220,7 @@ class Comment(UuidMixin, BaseMixin, db.Model): 'Comment', backref=sa.orm.backref('in_reply_to', remote_side='Comment.id') ) - _message = MarkdownColumnNative( # type: ignore[has-type] + _message = markdown_cached_column( # type: ignore[has-type] 'message', profile='basic', nullable=False ) diff --git a/funnel/models/helpers.py b/funnel/models/helpers.py index 1ac6d0502..350035e85 100644 --- a/funnel/models/helpers.py +++ b/funnel/models/helpers.py @@ -50,7 +50,7 @@ 'quote_autocomplete_like', 'ImgeeFurl', 'ImgeeType', - 'MarkdownColumnNative', + 'markdown_cached_column', ] RESERVED_NAMES: Set[str] = { @@ -594,7 +594,7 @@ def process_bind_param(self, value, dialect): return value -class MarkdownCompositeNative(MutableComposite): +class MarkdownCachedComposite(MutableComposite): """Represents Markdown text and rendered HTML as a composite column.""" profile: str @@ -647,7 +647,7 @@ def __json__(self) -> Dict[str, Optional[str]]: # Compare text value def __eq__(self, other): """Compare for equality.""" - return isinstance(other, MarkdownCompositeNative) and ( + return isinstance(other, MarkdownCachedComposite) and ( self.__composite_values__() == other.__composite_values__() ) @@ -680,7 +680,7 @@ def coerce(cls, key, value): return cls(value) -def markdown_column_native( +def markdown_cached_column( name: str, deferred: bool = False, group: Optional[str] = None, @@ -701,7 +701,7 @@ def markdown_column_native( # Construct a custom subclass of MarkdownComposite and set the markdown processor # and processor options on it. We'll pass this class to SQLAlchemy's composite # constructor. - class CustomMarkdownComposite(MarkdownCompositeNative): + class CustomMarkdownComposite(MarkdownCachedComposite): pass CustomMarkdownComposite.profile = profile @@ -713,6 +713,3 @@ class CustomMarkdownComposite(MarkdownCompositeNative): deferred=deferred, group=group or name, ) - - -MarkdownColumnNative = markdown_column_native diff --git a/funnel/models/update.py b/funnel/models/update.py index 95a234b17..e011be064 100644 --- a/funnel/models/update.py +++ b/funnel/models/update.py @@ -20,13 +20,13 @@ BaseScopedIdNameMixin, Commentset, Mapped, - MarkdownColumnNative, Project, TimestampMixin, TSVectorType, User, UuidMixin, db, + markdown_cached_column, sa, ) from .comment import SET_TYPE @@ -101,7 +101,7 @@ class Update(UuidMixin, BaseScopedIdNameMixin, TimestampMixin, db.Model): ) parent = sa.orm.synonym('project') - body = MarkdownColumnNative( # type: ignore[has-type] + body = markdown_cached_column( # type: ignore[has-type] 'body', profile='basic', nullable=False ) diff --git a/funnel/models/venue.py b/funnel/models/venue.py index 34a89669c..7ea290542 100644 --- a/funnel/models/venue.py +++ b/funnel/models/venue.py @@ -9,9 +9,9 @@ from . import ( BaseScopedNameMixin, CoordinatesMixin, - MarkdownColumnNative, UuidMixin, db, + markdown_cached_column, sa, ) from .helpers import reopen @@ -32,7 +32,7 @@ class Venue(UuidMixin, BaseScopedNameMixin, CoordinatesMixin, db.Model): grants_via={None: project_child_role_map}, ) parent = sa.orm.synonym('project') - description = MarkdownColumnNative( # type: ignore[has-type] + description = markdown_cached_column( # type: ignore[has-type] 'description', profile='basic', default='', nullable=False ) address1 = sa.Column(sa.Unicode(160), default='', nullable=False) @@ -113,7 +113,7 @@ class VenueRoom(UuidMixin, BaseScopedNameMixin, db.Model): grants_via={None: set(project_child_role_map.values())}, ) parent = sa.orm.synonym('venue') - description = MarkdownColumnNative( # type: ignore[has-type] + description = markdown_cached_column( # type: ignore[has-type] 'description', profile='basic', default='', nullable=False ) bgcolor = sa.Column(sa.Unicode(6), nullable=False, default='229922') From 1b868d380e3e16248c5448bcd9de56e2ae45cdba Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Sun, 30 Oct 2022 16:21:46 +0530 Subject: [PATCH 20/72] Classes for markdown profiles. #1480 --- funnel/assets/js/submission_form.js | 2 +- funnel/utils/markdown/base.py | 54 +++++++++----------- funnel/utils/markdown/profiles.py | 79 ++++++++++++++++------------- funnel/views/api/markdown.py | 15 +++--- 4 files changed, 75 insertions(+), 75 deletions(-) diff --git a/funnel/assets/js/submission_form.js b/funnel/assets/js/submission_form.js index c5606efb4..a90ccc50e 100644 --- a/funnel/assets/js/submission_form.js +++ b/funnel/assets/js/submission_form.js @@ -22,7 +22,7 @@ $(() => { 'Content-Type': 'application/x-www-form-urlencoded', }, body: new URLSearchParams({ - type: 'submission', + type: 'document', text: $('#body').val(), }).toString(), }); diff --git a/funnel/utils/markdown/base.py b/funnel/utils/markdown/base.py index 23aa92f92..522f9f1d0 100644 --- a/funnel/utils/markdown/base.py +++ b/funnel/utils/markdown/base.py @@ -1,7 +1,7 @@ """Base files for markdown parser.""" # pylint: disable=too-many-arguments -from typing import Any, Dict, List, Optional, Union, overload +from typing import List, Optional, overload import json from markdown_it import MarkdownIt @@ -33,18 +33,16 @@ @overload -def markdown(text: None, profile: Union[str, Dict[str, Any]]) -> None: +def markdown(text: None, profile: Optional[str]) -> None: ... @overload -def markdown(text: str, profile: Union[str, Dict[str, Any]]) -> Markup: +def markdown(text: str, profile: Optional[str]) -> Markup: ... -def markdown( - text: Optional[str], profile: Union[str, Dict[str, Any]] -) -> Optional[Markup]: +def markdown(text: Optional[str], profile: Optional[str]) -> Optional[Markup]: """ Markdown parser compliant with Commonmark+GFM using markdown-it-py. @@ -56,32 +54,25 @@ def markdown( # Replace invisible characters with spaces text = normalize_spaces_multiline(text) - if isinstance(profile, str): - try: - _profile = profiles[profile] - except KeyError as exc: - raise KeyError( - f'Wrong markdown config profile "{profile}". Check name.' - ) from exc - elif isinstance(profile, dict): - _profile = profile - else: - raise TypeError('Wrong type - profile has to be either str or dict') - - args = _profile.get('args', ()) - - md = MarkdownIt(*args) + try: + _profile = profiles[profile] + except KeyError as exc: + raise KeyError( + f'Wrong markdown config profile "{profile}". Check name.' + ) from exc - funnel_config = _profile.get('funnel_config', {}) + md = MarkdownIt(*_profile.args) if md.linkify is not None: md.linkify.set({'fuzzy_link': False, 'fuzzy_email': False}) for action in ['enableOnly', 'enable', 'disable']: - if action in funnel_config: - getattr(md, action)(funnel_config[action]) + if action in _profile.post_config: + getattr(md, action)( + _profile.post_config[action] # type: ignore[literal-required] + ) - for e in _profile.get('plugins', []): + for e in _profile.plugins: try: ext = plugins[e] except KeyError as exc: @@ -91,21 +82,22 @@ def markdown( md.use(ext, **plugin_configs.get(e, {})) # type: ignore[arg-type] - return Markup(getattr(md, funnel_config.get('render_with', 'render'))(text)) + return Markup(getattr(md, _profile.render_with)(text)) def _print_rules(md: MarkdownIt, active: str = None): """Debug function to be removed before merge.""" rules = {'all_rules': md.get_all_rules(), 'active_rules': {}} for p, pr in profiles.items(): - m = MarkdownIt(*pr.get('args', ())) - fc = pr.get('funnel_config', {}) + m = MarkdownIt(*pr.args) if m.linkify is not None: m.linkify.set({'fuzzy_link': False, 'fuzzy_email': False}) for action in ['enableOnly', 'enable', 'disable']: - if action in fc: - getattr(m, action)(fc[action]) - for e in pr.get('plugins', []): + if action in pr.post_config: + getattr(m, action)( + pr.post_config[action] # type: ignore[literal-required] + ) + for e in pr.plugins: try: ext = plugins[e] except KeyError as exc: diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 7eb7df66d..9cf7d7e72 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -1,8 +1,9 @@ """Config profiles for markdown parser.""" -from typing import Any, Callable, Dict +from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Type, TypedDict from mdit_py_plugins import anchors, footnote, tasklists +from typing_extensions import NotRequired from coaster.utils import make_name @@ -24,6 +25,7 @@ 'tasklists': {'enabled': True, 'label': True, 'label_after': False}, } + default_markdown_options = { 'html': False, 'linkify': True, @@ -32,40 +34,45 @@ } -# Config profiles. -# -# Format: { -# 'args': ( -# config:str | preset.make(), -# options_update: Mapping | None -# ), -# 'funnel_config' : { # Optional -# 'enable': List = [], -# 'disable': List = [], -# 'enableOnly': List = [], -# 'render_with': str = 'render' -# } -# } - -profiles: Dict[str, Dict[str, Any]] = { - 'basic': { - 'args': ('gfm-like', default_markdown_options), - 'plugins': [], - 'funnel_config': {'disable': ['table']}, - }, - 'document': { - 'args': ('gfm-like', default_markdown_options), - 'plugins': [ - 'footnote', - 'heading_anchors', - 'tasklists', +class PostConfig(TypedDict): + disable: NotRequired[List[str]] + enable: NotRequired[List[str]] + enableOnly: NotRequired[List[str]] # noqa: N815 + + +class MarkdownProfile: + args: Tuple[str, Mapping] = ('gfm-like', default_markdown_options) + plugins: List[str] = [] + post_config: PostConfig = {} + render_with: str = 'render' + + +class MarkdownProfileBasic(MarkdownProfile): + post_config: PostConfig = {'disable': ['table']} + + +class MarkdownProfileDocument(MarkdownProfile): + plugins: List[str] = [ + 'footnote', + 'heading_anchors', + 'tasklists', + ] + + +class MarkdownProfileTextField(MarkdownProfile): + args: Tuple[str, Mapping] = ('zero', default_markdown_options) + post_config: PostConfig = { + 'enable': [ + 'emphasis', + 'backticks', ], - }, - 'text-field': { - 'args': ('zero', {'breaks': False}), - 'funnel_config': { - 'enable': ['emphasis', 'backticks'], - 'render_with': 'renderInline', - }, - }, + } + render_with: str = 'renderInline' + + +profiles: Dict[Optional[str], Type[MarkdownProfile]] = { + None: MarkdownProfileDocument, + 'basic': MarkdownProfileBasic, + 'document': MarkdownProfileDocument, + 'text-field': MarkdownProfileTextField, } diff --git a/funnel/views/api/markdown.py b/funnel/views/api/markdown.py index e4102d44f..490553de5 100644 --- a/funnel/views/api/markdown.py +++ b/funnel/views/api/markdown.py @@ -1,27 +1,28 @@ """Markdown preview view.""" +from typing import Optional + from flask import request from ... import app from ...typing import ReturnView from ...utils import markdown - -# extra_markdown_types = {'profile', 'project', 'submission', 'session'} +from ...utils.markdown.profiles import profiles @app.route('/api/1/preview/markdown', methods=['POST']) def markdown_preview() -> ReturnView: """Render Markdown in the backend, with custom options based on use case.""" - # The `type` differentiator is temporarily not supported with new markdown - # mtype = request.form.get('type') + profile: Optional[str] = request.form.get('profile') + if profile not in profiles: + profile = None text = request.form.get('text') - html = markdown(text, 'document') + html = markdown(text, profile) return { 'status': 'ok', - # 'type': mtype if mtype in extra_markdown_types else None, - 'type': None, + 'profile': profile, 'html': html, } From b16a2e7146239f80c30e7e473ef43d039c52224e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 08:26:52 +0000 Subject: [PATCH 21/72] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- funnel/views/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funnel/views/project.py b/funnel/views/project.py index fa3c77b6a..78f6969a0 100644 --- a/funnel/views/project.py +++ b/funnel/views/project.py @@ -507,7 +507,7 @@ def edit_boxoffice_data(self) -> ReturnView: obj=SimpleNamespace( org=boxoffice_data.get('org', ''), item_collection_id=boxoffice_data.get('item_collection_id', ''), - allow_rsvp = self.obj.allow_rsvp + allow_rsvp=self.obj.allow_rsvp, ), model=Project, ) From 54fff748e7b6b3c746dcf426d878b24fb8674ac0 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 31 Oct 2022 18:19:04 +0530 Subject: [PATCH 22/72] Corrected preview bug - send profile to markdown API. Switched markdown() to accept pre-defined string key or a MarkdownProfile subclass as profile argument. #1480 --- funnel/assets/js/submission_form.js | 2 +- funnel/utils/markdown/base.py | 31 +++++++++++++++++++---------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/funnel/assets/js/submission_form.js b/funnel/assets/js/submission_form.js index a90ccc50e..9a589ada1 100644 --- a/funnel/assets/js/submission_form.js +++ b/funnel/assets/js/submission_form.js @@ -22,7 +22,7 @@ $(() => { 'Content-Type': 'application/x-www-form-urlencoded', }, body: new URLSearchParams({ - type: 'document', + profile: 'document', text: $('#body').val(), }).toString(), }); diff --git a/funnel/utils/markdown/base.py b/funnel/utils/markdown/base.py index 522f9f1d0..7d7efdf83 100644 --- a/funnel/utils/markdown/base.py +++ b/funnel/utils/markdown/base.py @@ -1,7 +1,7 @@ """Base files for markdown parser.""" # pylint: disable=too-many-arguments -from typing import List, Optional, overload +from typing import List, Optional, Type, Union, overload import json from markdown_it import MarkdownIt @@ -9,7 +9,7 @@ from coaster.utils.text import normalize_spaces_multiline -from .profiles import plugin_configs, plugins, profiles +from .profiles import MarkdownProfile, plugin_configs, plugins, profiles __all__ = ['markdown'] @@ -33,16 +33,18 @@ @overload -def markdown(text: None, profile: Optional[str]) -> None: +def markdown(text: None, profile: Optional[Union[str, Type[MarkdownProfile]]]) -> None: ... @overload -def markdown(text: str, profile: Optional[str]) -> Markup: +def markdown(text: str, profile: Optional[Union[str, Type[MarkdownProfile]]]) -> Markup: ... -def markdown(text: Optional[str], profile: Optional[str]) -> Optional[Markup]: +def markdown( + text: Optional[str], profile: Optional[Union[str, Type[MarkdownProfile]]] +) -> Optional[Markup]: """ Markdown parser compliant with Commonmark+GFM using markdown-it-py. @@ -54,12 +56,19 @@ def markdown(text: Optional[str], profile: Optional[str]) -> Optional[Markup]: # Replace invisible characters with spaces text = normalize_spaces_multiline(text) - try: - _profile = profiles[profile] - except KeyError as exc: - raise KeyError( - f'Wrong markdown config profile "{profile}". Check name.' - ) from exc + if profile is None or isinstance(profile, str): + try: + _profile = profiles[profile] + except KeyError as exc: + raise KeyError( + f'Wrong markdown config profile "{profile}". Check name.' + ) from exc + elif issubclass(profile, MarkdownProfile): + _profile = profile + else: + raise TypeError( + 'Wrong type - profile has to be either str or a subclass of MarkdownProfile' + ) md = MarkdownIt(*_profile.args) From 3ea2cfa99e3c1ac43368dcef3dbccce86af2b4fd Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 3 Nov 2022 14:25:41 +0530 Subject: [PATCH 23/72] Better class-based tests structure for markdown. #1490 --- tests/data/markdown/basic.toml | 1 - tests/data/markdown/code.toml | 1 - tests/data/markdown/footnotes.toml | 8 +- tests/data/markdown/headings.toml | 7 - tests/data/markdown/images.toml | 1 - tests/data/markdown/links.toml | 1 - tests/data/markdown/lists.toml | 1 - tests/data/markdown/tables.toml | 1 - tests/unit/utils/test_markdown.py | 371 +++++++++-------------------- 9 files changed, 110 insertions(+), 282 deletions(-) diff --git a/tests/data/markdown/basic.toml b/tests/data/markdown/basic.toml index 88b1fc638..1275157b5 100644 --- a/tests/data/markdown/basic.toml +++ b/tests/data/markdown/basic.toml @@ -1,4 +1,3 @@ -[data] markdown = """ ## Basic markup This is a sample paragraph that has **asterisk bold**, *asterisk emphasized*, __underscore bold__ and _underscore italic_ strings. diff --git a/tests/data/markdown/code.toml b/tests/data/markdown/code.toml index 14464fd8e..efe078f56 100644 --- a/tests/data/markdown/code.toml +++ b/tests/data/markdown/code.toml @@ -1,4 +1,3 @@ -[data] markdown = """ ## Code diff --git a/tests/data/markdown/footnotes.toml b/tests/data/markdown/footnotes.toml index 97cd3271f..e2ad3fe09 100644 --- a/tests/data/markdown/footnotes.toml +++ b/tests/data/markdown/footnotes.toml @@ -1,4 +1,3 @@ -[data] markdown = """ ## Footnotes Here is some random text! @@ -27,13 +26,8 @@ profiles = [ "basic", "document",] [config.custom_profiles.footnotes] args_config = "default" +args_options = {html = false, linkify = true, typographer = true, breaks = true} plugins = ["footnote"] -args = ["default", {html = false, linkify = true, typographer = true, breaks = true}] -[config.custom_profiles.footnotes.args_options_update] -html = false -linkify = true -typographer = true -breaks = true [expected_output] basic = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[^first].

\n

Footnote 2 link[^second].

\n

Inline footnote^[Text of inline footnote] definition.

\n

Duplicated footnote reference[^second].

\n

[^first]: Footnote can have markup

\n
and multiple paragraphs.\n
\n

[^second]: Footnote text.

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

" diff --git a/tests/data/markdown/headings.toml b/tests/data/markdown/headings.toml index 167570deb..4b19d00e5 100644 --- a/tests/data/markdown/headings.toml +++ b/tests/data/markdown/headings.toml @@ -1,4 +1,3 @@ -[data] markdown = """ Using the heading-anchors plugin with it's default config: @@ -30,12 +29,6 @@ profiles = [ "basic", "document",] [config.custom_profiles.heading_anchors] args_config = "default" plugins = [ "heading_anchors",] -args = ["default", {html = false, linkify = true, typographer = true, breaks = true}] -[config.custom_profiles.heading_anchors.args_options_update] -html = false -linkify = true -typographer = true -breaks = true [expected_output] basic = "

Using the heading-anchors plugin with it's default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" diff --git a/tests/data/markdown/images.toml b/tests/data/markdown/images.toml index 3cb85559f..dc8958180 100644 --- a/tests/data/markdown/images.toml +++ b/tests/data/markdown/images.toml @@ -1,4 +1,3 @@ -[data] markdown = """ ## Images diff --git a/tests/data/markdown/links.toml b/tests/data/markdown/links.toml index 4079302bc..011d62eb8 100644 --- a/tests/data/markdown/links.toml +++ b/tests/data/markdown/links.toml @@ -1,4 +1,3 @@ -[data] markdown = """ ## Links diff --git a/tests/data/markdown/lists.toml b/tests/data/markdown/lists.toml index b59f25d0c..057ef157b 100644 --- a/tests/data/markdown/lists.toml +++ b/tests/data/markdown/lists.toml @@ -1,4 +1,3 @@ -[data] markdown = """ ## Lists diff --git a/tests/data/markdown/tables.toml b/tests/data/markdown/tables.toml index 944f58cdd..3e5f7db98 100644 --- a/tests/data/markdown/tables.toml +++ b/tests/data/markdown/tables.toml @@ -1,4 +1,3 @@ -[data] markdown = """ ## Tables diff --git a/tests/unit/utils/test_markdown.py b/tests/unit/utils/test_markdown.py index bd7915cc3..34247be6e 100644 --- a/tests/unit/utils/test_markdown.py +++ b/tests/unit/utils/test_markdown.py @@ -1,289 +1,136 @@ """Tests for markdown parser.""" -from copy import copy, deepcopy -from datetime import datetime -from functools import lru_cache from pathlib import Path -from typing import Any, Callable, Dict, List, Tuple, Union -import json +from typing import Dict, List, Optional, Type +import warnings -from bs4 import BeautifulSoup -from markupsafe import Markup import pytest import tomlkit -from funnel.utils import markdown -from funnel.utils.markdown.profiles import default_markdown_options, profiles +from funnel.utils.markdown import markdown +from funnel.utils.markdown.profiles import MarkdownProfile, profiles +DATAROOT: Path = Path('tests/data/markdown') +DEBUG = True -def markdown_fn(fn: str) -> Callable: - dataroot: Path = Path('tests/data/markdown') - case_type = Dict[str, Any] - cases_type = Dict[str, case_type] +class MarkdownCase: + def __init__( + self, + test_id: str, + markdown: str, + profile_id: str, + profile: Optional[Dict] = None, + expected_output: Optional[str] = None, + ) -> None: + self.test_id: str = test_id + self.markdown: str = markdown + self.profile_id: str = profile_id + self.profile: Optional[Type[MarkdownProfile]] = MarkdownCase.make_profile( + profile + ) + self.expected_output: Optional[str] = expected_output - @lru_cache() - def load_md_cases() -> cases_type: - """Load test cases for the markdown parser from .toml files.""" - return { - file.name: tomlkit.loads(file.read_text()) - for file in dataroot.iterdir() - if file.suffix == '.toml' - } + if self.is_custom() and self.profile_id in profiles: + raise Exception( + f'Case {self.case_id}: Custom profiles cannot use a key that is pre-defined in profiles' + ) - def get_case_profiles(case: case_type) -> Dict[str, Any]: - """Return dict with key-value of profiles for the provided test case.""" - profiles_out: Dict[str, Any] = {} - case_profiles = copy(case['config']['profiles']) - _profiles = deepcopy(profiles) - if 'custom_profiles' in case['config']: - for p in case['config']['custom_profiles']: - if p not in _profiles: - _profiles[p] = case['config']['custom_profiles'][p] - if _profiles[p]['args_options_update'] is False: - _profiles[p]['args_options_update'] = default_markdown_options - _profiles[p]['args'] = ( - _profiles[p]['args_config'], - _profiles[p]['args_options_update'], - ) - if p not in case_profiles: - case_profiles.append(p) - for p in case_profiles: - if p in _profiles: - profiles_out[p] = _profiles[p] - else: - profiles_out[p] = None - return profiles_out + @staticmethod + def make_profile(profile: Optional[Dict]): + if profile is None: + return None + + class MarkdownProfileCustom(MarkdownProfile): + pass - def get_md(case: case_type, profile: Union[str, Dict[str, Any]]): - """Parse a markdown test case for given configuration profile.""" + l: List = list(MarkdownProfileCustom.args) + if 'args_config' in profile: + l[0] = profile['args_config'] + if 'args_options' in profile: + l[1].update(profile['args_options']) + MarkdownProfileCustom.args = (l[0], l[1]) + if 'plugins' in profile: + MarkdownProfileCustom.plugins = profile['plugins'] + if 'post_config' in profile: + MarkdownProfileCustom.post_config = profile['post_config'] + if 'render_with' in profile: + MarkdownProfileCustom.render_with = profile['render_with'] + return MarkdownProfileCustom + + def __repr__(self) -> str: + return self.case_id + + def is_custom(self): + return self.profile is not None + + @property + def case_id(self) -> str: + return f'{self.test_id}-{self.profile_id}' + + @property + def markdown_profile(self): + return self.profile if self.profile is not None else self.profile_id + + @property + def output(self): return ( - markdown( # pylint: disable=unnecessary-dunder-call - case['data']['markdown'], profile - ) + markdown(self.markdown, self.markdown_profile) .__str__() .lstrip('\n\r') .rstrip(' \n\r') ) - def update_expected_case_output(cases: cases_type, debug: bool) -> None: - """Update cases object with expected output for each case-profile combination.""" - for case_id in cases: - case = cases[case_id] - _profiles = get_case_profiles(case) - case['expected_output'] = {} - for profile_id, _profile in _profiles.items(): - case['expected_output'][profile_id] = get_md( - case, - profiles[profile_id] if profile_id in profiles else _profile, - ) - - def dump_md_cases(cases: cases_type) -> None: - """Save test cases for the markdown parser to .toml files.""" - for (file, file_data) in cases.items(): - (dataroot / file).write_text(tomlkit.dumps(file_data)) - - def update_test_data(debug: bool = False) -> None: - """Update test data after changes made to test cases and/or configurations.""" - cases = load_md_cases() - update_expected_case_output(cases, debug) - dump_md_cases(cases) - - @lru_cache() - def get_output_template() -> BeautifulSoup: - """Get bs4 output template for output.html for markdown tests.""" - return BeautifulSoup((dataroot / 'template.html').read_text(), 'html.parser') - @lru_cache() - def get_case_template() -> BeautifulSoup: - """Get blank bs4 template for each case to be used to update test output.""" - return get_output_template().find(id='output_template') - - # pylint: disable=too-many-arguments - def update_case_output( - case: case_type, - profile_id: str, - output: str, - ) -> None: - """Update case with output for provided case-configuration.""" - if 'output' not in case: - case['output'] = {} - case['output'][profile_id] = output - - def update_case_output_template( - template: BeautifulSoup, - case_id: str, - case: case_type, - profile_id: str, - profile: Dict[str, Any], - output: str, - ) -> None: - """Update case and output template with output for provided case-configuration.""" - profile = deepcopy(profile) - if 'output' not in case: - case['output'] = {} - case['output'][profile_id] = output - op = copy(get_case_template()) - del op['id'] - op.select('.filename')[0].string = case_id - op.select('.profile')[0].string = str(profile_id) - if 'args_config' in profile: - del profile['args_config'] - if 'args_options_update' in profile: - del profile['args_options_update'] - op.select('.config')[0].string = json.dumps(profile, indent=2) - op.select('.markdown .output')[0].append(case['data']['markdown']) - try: - expected_output = case['expected_output'][profile_id] - except KeyError: - expected_output = markdown( - f'Expected output for `{case_id}` config `{profile_id}` ' - 'has not been generated. Please run `make tests-data-md`' - '**after evaluating other failures**.\n' - '`make tests-data-md` should only be run for this after ' - 'ensuring there are no unexpected mismatches/failures ' - 'in the output of all cases.\n' - 'For detailed instructions check the [readme](readme.md).', - 'basic', - ) - op.select('.expected .output')[0].append( - BeautifulSoup(expected_output, 'html.parser') - ) - op.select('.final_output .output')[0].append( - BeautifulSoup(output, 'html.parser') - ) - op['class'] = op.get('class', []) + [ - 'success' if expected_output == output and profile is not None else 'failed' - ] - template.find('body').append(op) - - def dump_md_output(output: BeautifulSoup) -> None: - """Save test output in output.html.""" - output.find(id='generated').string = datetime.now().strftime( - '%d %B, %Y %H:%M:%S' - ) - (dataroot / 'output.html').write_text(output.prettify()) - - @lru_cache() - def get_md_test_data(debug: bool) -> cases_type: - """Get cases updated with final output alongwith test cases dataset.""" - if debug: - template = get_output_template() - cases = load_md_cases() - for case_id, case in cases.items(): - _profiles = get_case_profiles(case) - for profile_id, profile in _profiles.items(): - test_output = get_md(case, profile) - if debug: - update_case_output_template( - template, - case_id, - case, +class MarkdownTestRegistry: + test_map: Optional[Dict[str, Dict[str, MarkdownCase]]] = None + + @classmethod + def load(cls): + if cls.test_map is None: + cls.test_map = {} + tests = { + file.name: tomlkit.loads(file.read_text()) + for file in DATAROOT.iterdir() + if file.suffix == '.toml' + } + for test_id, test in tests.items(): + config = test['config'] + exp = test.get('expected_output', {}) + cls.test_map[test_id] = { + profile_id: MarkdownCase( + test_id, + test['markdown'], profile_id, - profile, - test_output, + profile=profile, + expected_output=exp.get(profile_id, None), ) - else: - update_case_output(case, profile_id, test_output) - if debug: - dump_md_output(template) - return cases - - def get_md_test_dataset() -> List[Tuple[str, str]]: - """Return testcase datasets.""" - return [ - (case_id, profile_id) - for (case_id, case) in load_md_cases().items() - for profile_id in get_case_profiles(case) - ] - - @lru_cache() - def get_md_test_output( - case_id: str, profile_id: str, debug: bool = False - ) -> Tuple[str, str]: - """Return expected output and final output for quoted case-config combination.""" - cases = get_md_test_data(debug) - if not debug: - expected_output = cases[case_id]['expected_output'][profile_id] - output = cases[case_id]['output'][profile_id] - else: - try: - expected_output = cases[case_id]['expected_output'][profile_id] - output = cases[case_id]['output'][profile_id] - except KeyError: - return ( - f'Expected output for "{case_id}" config "{profile_id}" has not been generated. ' - 'Please run "make tests-data-md" ' - 'after evaluating other failures.\n' - '"make tests-data-md" should only be run for this after ' - 'ensuring there are no unexpected mismatches/failures ' - 'in the output of all cases.\n', - 'For detailed instructions check "tests/data/markdown/readme.md". \n' - + ( - cases[case_id]['output'][profile_id] - if len(cases[case_id]['output'][profile_id]) <= 160 - else '\n'.join( - [ - cases[case_id]['output'][profile_id][:80], - '...', - cases[case_id]['output'][profile_id][-80:], - ] - ) - ), - ) - return (expected_output, output) - - if fn == 'get_dataset': - return get_md_test_dataset - elif fn == 'get_data': - return get_md_test_output - elif fn == 'update_data': - return update_test_data - else: - return lambda x: None - - -@pytest.fixture() -def update_markdown_tests_data(): - return markdown_fn('update_data') - - -@pytest.fixture(scope='session') -def markdown_output(): - return markdown_fn('get_data') - - -def test_markdown_none() -> None: - assert markdown(None, 'basic') is None - assert markdown(None, 'document') is None - - -def test_markdown_blank() -> None: - assert markdown('', 'basic') == Markup('') - assert markdown('', 'document') == Markup('') - - -@pytest.mark.update_markdown_data() -def test_markdown_update_output(pytestconfig, update_markdown_tests_data): - has_mark = pytestconfig.getoption('-m', default=None) == 'update_markdown_data' - if not has_mark: - pytest.skip('Skipping update of expected output of markdown test cases') - update_markdown_tests_data(debug=False) + for profile_id, profile in { + **{p: None for p in config.get('profiles', [])}, + **config.get('custom_profiles', {}), + }.items() + } + + @classmethod + def dataset(cls) -> List[MarkdownCase]: + cls.load() + return ( + [case for tests in cls.test_map.values() for case in tests.values()] + if cls.test_map is not None + else [] + ) @pytest.mark.parametrize( - ( - 'case_id', - 'profile_id', - ), - markdown_fn('get_dataset')(), + 'case', + MarkdownTestRegistry.dataset(), ) -def test_markdown_dataset( - case_id: str, profile_id: str, markdown_output, unified_diff_output -) -> None: - debug = False - (expected_output, output) = markdown_output(case_id, profile_id, debug=debug) - if debug: - unified_diff_output(expected_output, output) +def test_markdown_cases(case: MarkdownCase, unified_diff_output) -> None: + if case.expected_output is None: + warnings.warn(f'Expected output not generated for {case}') + pytest.skip(f'Expected output not generated for {case}') + + if DEBUG: + unified_diff_output(case.expected_output, case.output) else: - assert expected_output == output + assert case.expected_output == case.output From 73c692decdbb003d302607d877bcf3c9e30be966 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 3 Nov 2022 17:29:11 +0530 Subject: [PATCH 24/72] Functionality to update expected output for markdown tests. #1490 Resolves #1506 --- tests/data/markdown/basic.toml | 122 ++++++++++++++++++++++++++++- tests/data/markdown/code.toml | 98 ++++++++++++++++++++++- tests/data/markdown/footnotes.toml | 59 +++++++++++++- tests/data/markdown/headings.toml | 60 +++++++++++++- tests/data/markdown/images.toml | 22 +++++- tests/data/markdown/links.toml | 104 +++++++++++++++++++++++- tests/data/markdown/lists.toml | 104 +++++++++++++++++++++++- tests/data/markdown/tables.toml | 62 ++++++++++++++- tests/unit/utils/test_markdown.py | 55 +++++++++---- 9 files changed, 651 insertions(+), 35 deletions(-) diff --git a/tests/data/markdown/basic.toml b/tests/data/markdown/basic.toml index 1275157b5..78ff9d637 100644 --- a/tests/data/markdown/basic.toml +++ b/tests/data/markdown/basic.toml @@ -78,5 +78,123 @@ ___ profiles = [ "basic", "document",] [expected_output] -basic = "

Basic markup

\n

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

\n

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
\nIt also has a newline break here!!!

\n

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

\n

__Bold without closure does not work
\n**Bold without closure does not work
\n_Emphasis without closure does not work
\n*Emphasis without closure does not work

\n

Bold without closure
\non the same line
\ncarries forward to
\nthis is text that should strike off
\nmultiple consecutive lines

\n

Bold without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Horizontal Rules

\n
\n

Above is a horizontal rule using hyphens.
\nThis is text that should strike off

\n
\n

Above is a horizontal rule using asterisks.
\nBelow is a horizontal rule using underscores.

\n
\n

Links

\n

Here is a link to hasgeek

\n

Link to funnel with the title 'Hasgeek'

\n

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

\n
\n

Markdown-it typography

\n

The results of the below depend on the typographer options enabled by us in the markdown-it-py parser, if typographer=True has been passed to it.

\n

The below should convert if replacements has been enabled.

\n

(c) (C) (r) (R) (tm) (TM) (p) (P) +-
\ntest.. test... test..... test?..... test!....
\n!!!!!! ???? ,, -- ---

\n

The below should convert the quotes if smartquotes has been enabled.

\n

"Smartypants, double quotes" and 'single quotes'

\n
\n

Blockquotes

\n
\n

Blockquotes can also be nested...

\n
\n

...by using additional greater-than signs right next to each other...

\n
\n

...or with spaces between arrows.

\n
\n
\n
\n
" -document = "

Basic markup

\n

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

\n

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
\nIt also has a newline break here!!!

\n

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

\n

__Bold without closure does not work
\n**Bold without closure does not work
\n_Emphasis without closure does not work
\n*Emphasis without closure does not work

\n

Bold without closure
\non the same line
\ncarries forward to
\nthis is text that should strike off
\nmultiple consecutive lines

\n

Bold without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Emphasis without closure
\non the same line
\ncarries forward to
\nmultiple consecutive lines

\n

Horizontal Rules

\n
\n

Above is a horizontal rule using hyphens.
\nThis is text that should strike off

\n
\n

Above is a horizontal rule using asterisks.
\nBelow is a horizontal rule using underscores.

\n
\n

Links

\n

Here is a link to hasgeek

\n

Link to funnel with the title 'Hasgeek'

\n

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

\n
\n

Markdown-it typography

\n

The results of the below depend on the typographer options enabled by us in the markdown-it-py parser, if typographer=True has been passed to it.

\n

The below should convert if replacements has been enabled.

\n

(c) (C) (r) (R) (tm) (TM) (p) (P) +-
\ntest.. test... test..... test?..... test!....
\n!!!!!! ???? ,, -- ---

\n

The below should convert the quotes if smartquotes has been enabled.

\n

"Smartypants, double quotes" and 'single quotes'

\n
\n

Blockquotes

\n
\n

Blockquotes can also be nested...

\n
\n

...by using additional greater-than signs right next to each other...

\n
\n

...or with spaces between arrows.

\n
\n
\n
\n
" +basic = """

Basic markup

+

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

+

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
+It also has a newline break here!!!

+

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

+

__Bold without closure does not work
+**Bold without closure does not work
+_Emphasis without closure does not work
+*Emphasis without closure does not work

+

Bold without closure
+on the same line
+carries forward to
+this is text that should strike off
+multiple consecutive lines

+

Bold without closure
+on the same line
+carries forward to
+multiple consecutive lines

+

Emphasis without closure
+on the same line
+carries forward to
+multiple consecutive lines

+

Emphasis without closure
+on the same line
+carries forward to
+multiple consecutive lines

+

Horizontal Rules

+
+

Above is a horizontal rule using hyphens.
+This is text that should strike off

+
+

Above is a horizontal rule using asterisks.
+Below is a horizontal rule using underscores.

+
+

Links

+

Here is a link to hasgeek

+

Link to funnel with the title 'Hasgeek'

+

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

+
+

Markdown-it typography

+

The results of the below depend on the typographer options enabled by us in the markdown-it-py parser, if typographer=True has been passed to it.

+

The below should convert if replacements has been enabled.

+

(c) (C) (r) (R) (tm) (TM) (p) (P) +-
+test.. test... test..... test?..... test!....
+!!!!!! ???? ,, -- ---

+

The below should convert the quotes if smartquotes has been enabled.

+

"Smartypants, double quotes" and 'single quotes'

+
+

Blockquotes

+
+

Blockquotes can also be nested...

+
+

...by using additional greater-than signs right next to each other...

+
+

...or with spaces between arrows.

+
+
+
+
+""" +document = """

Basic markup

+

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

+

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
+It also has a newline break here!!!

+

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

+

__Bold without closure does not work
+**Bold without closure does not work
+_Emphasis without closure does not work
+*Emphasis without closure does not work

+

Bold without closure
+on the same line
+carries forward to
+this is text that should strike off
+multiple consecutive lines

+

Bold without closure
+on the same line
+carries forward to
+multiple consecutive lines

+

Emphasis without closure
+on the same line
+carries forward to
+multiple consecutive lines

+

Emphasis without closure
+on the same line
+carries forward to
+multiple consecutive lines

+

Horizontal Rules

+
+

Above is a horizontal rule using hyphens.
+This is text that should strike off

+
+

Above is a horizontal rule using asterisks.
+Below is a horizontal rule using underscores.

+
+ +

Here is a link to hasgeek

+

Link to funnel with the title 'Hasgeek'

+

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

+
+

Markdown-it typography

+

The results of the below depend on the typographer options enabled by us in the markdown-it-py parser, if typographer=True has been passed to it.

+

The below should convert if replacements has been enabled.

+

(c) (C) (r) (R) (tm) (TM) (p) (P) +-
+test.. test... test..... test?..... test!....
+!!!!!! ???? ,, -- ---

+

The below should convert the quotes if smartquotes has been enabled.

+

"Smartypants, double quotes" and 'single quotes'

+
+

Blockquotes

+
+

Blockquotes can also be nested...

+
+

...by using additional greater-than signs right next to each other...

+
+

...or with spaces between arrows.

+
+
+
+
+""" diff --git a/tests/data/markdown/code.toml b/tests/data/markdown/code.toml index efe078f56..3ebee0aff 100644 --- a/tests/data/markdown/code.toml +++ b/tests/data/markdown/code.toml @@ -63,5 +63,99 @@ Isn't that **fantastic**! profiles = [ "basic", "document",] [expected_output] -basic = "

Code

\n

Inline code

\n

Indented code

\n
// Some comments\nline 1 of code\nline 2 of code\nline 3 of code\n
\n

Block code "fences"

\n
Sample text here...\nIt is a sample text that has multiple lines\n
\n

Syntax highlighting

\n

Javascript

\n
var foo = function (bar) {\n  return bar++;\n};\n\nconsole.log(foo(5));\nconsole.log('`This should be printed`');\n
\n
\n

Javascript can be highlighted by using either of the two keywords js and javascript

\n
\n

Python

\n
import os\nfrom funnel.utils.markdown import DATAROOT, markdown\n\nif os.file.path.exists(\n    os.file.path.join(\n        DATAROOT,\n        'file',\n        'path'\n    )\n):\n    markdown('# I can output ``` also with a \\!')\n
\n

Markdown

\n
*I can also type markdown code blocks.*\nIsn't that **fantastic**!\n\n- This is a list\n  - Just testing\n  - this out\n\n[hasgeek](https://hasgeek.com)\n
" -document = "

Code

\n

Inline code

\n

Indented code

\n
// Some comments\nline 1 of code\nline 2 of code\nline 3 of code\n
\n

Block code "fences"

\n
Sample text here...\nIt is a sample text that has multiple lines\n
\n

Syntax highlighting

\n

Javascript

\n
var foo = function (bar) {\n  return bar++;\n};\n\nconsole.log(foo(5));\nconsole.log('`This should be printed`');\n
\n
\n

Javascript can be highlighted by using either of the two keywords js and javascript

\n
\n

Python

\n
import os\nfrom funnel.utils.markdown import DATAROOT, markdown\n\nif os.file.path.exists(\n    os.file.path.join(\n        DATAROOT,\n        'file',\n        'path'\n    )\n):\n    markdown('# I can output ``` also with a \\!')\n
\n

Markdown

\n
*I can also type markdown code blocks.*\nIsn't that **fantastic**!\n\n- This is a list\n  - Just testing\n  - this out\n\n[hasgeek](https://hasgeek.com)\n
" +basic = """

Code

+

Inline code

+

Indented code

+
// Some comments
+line 1 of code
+line 2 of code
+line 3 of code
+
+

Block code "fences"

+
Sample text here...
+It is a sample text that has multiple lines
+
+

Syntax highlighting

+

Javascript

+
var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+console.log('`This should be printed`');
+
+
+

Javascript can be highlighted by using either of the two keywords js and javascript

+
+

Python

+
import os
+from funnel.utils.markdown import DATAROOT, markdown
+
+if os.file.path.exists(
+    os.file.path.join(
+        DATAROOT,
+        'file',
+        'path'
+    )
+):
+    markdown('# I can output ``` also with a \\!')
+
+

Markdown

+
*I can also type markdown code blocks.*
+Isn't that **fantastic**!
+
+- This is a list
+  - Just testing
+  - this out
+
+[hasgeek](https://hasgeek.com)
+
+""" +document = """

Code

+

Inline code

+

Indented code

+
// Some comments
+line 1 of code
+line 2 of code
+line 3 of code
+
+

Block code "fences"

+
Sample text here...
+It is a sample text that has multiple lines
+
+

Syntax highlighting

+

Javascript

+
var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+console.log('`This should be printed`');
+
+
+

Javascript can be highlighted by using either of the two keywords js and javascript

+
+

Python

+
import os
+from funnel.utils.markdown import DATAROOT, markdown
+
+if os.file.path.exists(
+    os.file.path.join(
+        DATAROOT,
+        'file',
+        'path'
+    )
+):
+    markdown('# I can output ``` also with a \\!')
+
+

Markdown

+
*I can also type markdown code blocks.*
+Isn't that **fantastic**!
+
+- This is a list
+  - Just testing
+  - this out
+
+[hasgeek](https://hasgeek.com)
+
+""" diff --git a/tests/data/markdown/footnotes.toml b/tests/data/markdown/footnotes.toml index e2ad3fe09..728cdb474 100644 --- a/tests/data/markdown/footnotes.toml +++ b/tests/data/markdown/footnotes.toml @@ -26,10 +26,61 @@ profiles = [ "basic", "document",] [config.custom_profiles.footnotes] args_config = "default" -args_options = {html = false, linkify = true, typographer = true, breaks = true} plugins = ["footnote"] [expected_output] -basic = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[^first].

\n

Footnote 2 link[^second].

\n

Inline footnote^[Text of inline footnote] definition.

\n

Duplicated footnote reference[^second].

\n

[^first]: Footnote can have markup

\n
and multiple paragraphs.\n
\n

[^second]: Footnote text.

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

" -document = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[1].

\n

Footnote 2 link[2].

\n

Inline footnote[3] definition.

\n

Duplicated footnote reference[2:1].

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

\n
\n
\n
    \n

  1. Footnote can have markup

    \n

    and multiple paragraphs. ↩︎

    \n
    2. \n

  2. Footnote text. ↩︎ ↩︎

    \n
    3. \n

  3. Text of inline footnote ↩︎

    \n
    4. \n
\n
" -footnotes = "

Footnotes

\n

Here is some random text!

\n

Footnote 1 link[1].

\n

Footnote 2 link[2].

\n

Inline footnote[3] definition.

\n

Duplicated footnote reference[2:1].

\n
\n

This is some more random text to test whether the footnotes are placed after this text.

\n
\n
\n
    \n

  1. Footnote can have markup

    \n

    and multiple paragraphs. ↩︎

    \n
    2. \n

  2. Footnote text. ↩︎ ↩︎

    \n
    3. \n

  3. Text of inline footnote ↩︎

    \n
    4. \n
\n
" +basic = """

Footnotes

+

Here is some random text!

+

Footnote 1 link[^first].

+

Footnote 2 link[^second].

+

Inline footnote^[Text of inline footnote] definition.

+

Duplicated footnote reference[^second].

+

[^first]: Footnote can have markup

+
and multiple paragraphs.
+
+

[^second]: Footnote text.

+
+

This is some more random text to test whether the footnotes are placed after this text.

+""" +document = """

Footnotes

+

Here is some random text!

+

Footnote 1 link[1].

+

Footnote 2 link[2].

+

Inline footnote[3] definition.

+

Duplicated footnote reference[2:1].

+
+

This is some more random text to test whether the footnotes are placed after this text.

+
+
+
    +

  1. Footnote can have markup

    +

    and multiple paragraphs. ↩︎

    +
    2. +

  2. Footnote text. ↩︎ ↩︎

    +
    3. +

  3. Text of inline footnote ↩︎

    +
    4. +
+
+""" +footnotes = """

Footnotes

+

Here is some random text!

+

Footnote 1 link[1].

+

Footnote 2 link[2].

+

Inline footnote[3] definition.

+

Duplicated footnote reference[2:1].

+
+

This is some more random text to test whether the footnotes are placed after this text.

+
+
+
    +

  1. Footnote can have markup

    +

    and multiple paragraphs. ↩︎

    +
    2. +

  2. Footnote text. ↩︎ ↩︎

    +
    3. +

  3. Text of inline footnote ↩︎

    +
    4. +
+
+""" diff --git a/tests/data/markdown/headings.toml b/tests/data/markdown/headings.toml index 4b19d00e5..b4c962431 100644 --- a/tests/data/markdown/headings.toml +++ b/tests/data/markdown/headings.toml @@ -31,6 +31,60 @@ args_config = "default" plugins = [ "heading_anchors",] [expected_output] -basic = "

Using the heading-anchors plugin with it's default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" -document = "

Using the heading-anchors plugin with it's default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" -heading_anchors = "

Using the heading-anchors plugin with it’s default config:

\n

The below headings should convert and get linked.
\nOnly the specified headings in extension defaults should get linked.

\n

h1 Heading

\n

h2 Heading

\n

h3 Heading

\n

h4 Heading

\n
h5 Heading
\n
h6 Heading
\n
\n

The below headings should not convert due to lack of space between markup and text
\n#h1 Heading
\n##h2 Heading
\n###h3 Heading
\n####h4 Heading
\n#####h5 Heading
\n######h6 Heading

\n

Text with 2 or more hyphens below it converts to H2

" +basic = """

Using the heading-anchors plugin with it's default config:

+

The below headings should convert and get linked.
+Only the specified headings in extension defaults should get linked.

+

h1 Heading

+

h2 Heading

+

h3 Heading

+

h4 Heading

+
h5 Heading
+
h6 Heading
+
+

The below headings should not convert due to lack of space between markup and text
+#h1 Heading
+##h2 Heading
+###h3 Heading
+####h4 Heading
+#####h5 Heading
+######h6 Heading

+

Text with 2 or more hyphens below it converts to H2

+""" +document = """

Using the heading-anchors plugin with it's default config:

+

The below headings should convert and get linked.
+Only the specified headings in extension defaults should get linked.

+

h1 Heading

+

h2 Heading

+

h3 Heading

+

h4 Heading

+
h5 Heading
+
h6 Heading
+
+

The below headings should not convert due to lack of space between markup and text
+#h1 Heading
+##h2 Heading
+###h3 Heading
+####h4 Heading
+#####h5 Heading
+######h6 Heading

+

Text with 2 or more hyphens below it converts to H2

+""" +heading_anchors = """

Using the heading-anchors plugin with it’s default config:

+

The below headings should convert and get linked.
+Only the specified headings in extension defaults should get linked.

+

h1 Heading

+

h2 Heading

+

h3 Heading

+

h4 Heading

+
h5 Heading
+
h6 Heading
+
+

The below headings should not convert due to lack of space between markup and text
+#h1 Heading
+##h2 Heading
+###h3 Heading
+####h4 Heading
+#####h5 Heading
+######h6 Heading

+

Text with 2 or more hyphens below it converts to H2

+""" diff --git a/tests/data/markdown/images.toml b/tests/data/markdown/images.toml index dc8958180..fbcdb58c3 100644 --- a/tests/data/markdown/images.toml +++ b/tests/data/markdown/images.toml @@ -24,5 +24,23 @@ With a reference later in the document defining the URL location. profiles = [ "basic", "document",] [expected_output] -basic = "

Images

\n

\"Logo\"
\n\"Logo\" Images stay inline within the same block after a new line

\n

\"The

\n

Like links, images also have a footnote style syntax

\n

\"Alt:
\n\"Alt:
\n\"Alt:

\n

With a reference later in the document defining the URL location.

" -document = "

Images

\n

\"Logo\"
\n\"Logo\" Images stay inline within the same block after a new line

\n

\"The

\n

Like links, images also have a footnote style syntax

\n

\"Alt:
\n\"Alt:
\n\"Alt:

\n

With a reference later in the document defining the URL location.

" +basic = """

Images

+

Logo
+Logo Images stay inline within the same block after a new line

+

The past as a compass for the future

+

Like links, images also have a footnote style syntax

+

Alt: Find your peers
+Alt: Discover your community
+Alt: Sustain the conversations

+

With a reference later in the document defining the URL location.

+""" +document = """

Images

+

Logo
+Logo Images stay inline within the same block after a new line

+

The past as a compass for the future

+

Like links, images also have a footnote style syntax

+

Alt: Find your peers
+Alt: Discover your community
+Alt: Sustain the conversations

+

With a reference later in the document defining the URL location.

+""" diff --git a/tests/data/markdown/links.toml b/tests/data/markdown/links.toml index 011d62eb8..b1760c7c2 100644 --- a/tests/data/markdown/links.toml +++ b/tests/data/markdown/links.toml @@ -63,5 +63,105 @@ _http://danlec_@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoA profiles = [ "basic", "document",] [expected_output] -basic = "

Links

\n

Hasgeek
\nHasgeek TV

\n

Autoconverted links

\n

https://github.com/nodeca/pica
\nhttp://twitter.com/hasgeek

\n

Footnote style syntax

\n

Hasgeek

\n

Links can have a footnote style syntax, where the links can be defined later. This is helpful in the case of a repetitive URL that needs to be linked to.

\n

Unsafe links

\n

These are links that should not convert (source)

\n

[a](javascript:prompt(document.cookie))
\n[a](j a v a s c r i p t:prompt(document.cookie))
\n![a](javascript:prompt(document.cookie))
\n<javascript:prompt(document.cookie)>
\n<&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29>
\n![a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\na
\n![a'"`onerror=prompt(document.cookie)](x)
\n[citelol]: (javascript:prompt(document.cookie))
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[test](javascript://%0d%0aprompt(1))
\n[test](javascript://%0d%0aprompt(1);com)
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[notmalicious](javascript://%0d%0awindow.onerror=alert;throw%20document.cookie)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[clickme](vbscript:alert(document.domain))
\nhttp://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
\ntext
\n[a](javascript:this;alert(1))
\n[a](javascript:this;alert(1))
\n[a](Javascript:alert(1))
\n[a](Javas%26%2399;ript:alert(1))
\n[a](javascript:alert�(1))
\n[a](javascript:confirm(1)
\n[a](javascript://www.google.com%0Aprompt(1))
\n[a](javascript://%0d%0aconfirm(1);com)
\n[a](javascript:window.onerror=confirm;throw%201)
\n[a](x01javascript:alert(document.domain))
\n[a](javascript://www.google.com%0Aalert(1))
\na
\n[a](JaVaScRiPt:alert(1))
\n\"a\"
\n\"a\"
\n</http://<?php><\\h1>script:scriptconfirm(2)
\nXSS
\n[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
\n[ ](http://a?p=[[/onclick=alert(0) .]])
\n[a](javascript:new%20Function`al\\ert`1``;)

" -document = "

Links

\n

Hasgeek
\nHasgeek TV

\n

Autoconverted links

\n

https://github.com/nodeca/pica
\nhttp://twitter.com/hasgeek

\n

Footnote style syntax

\n

Hasgeek

\n

Links can have a footnote style syntax, where the links can be defined later. This is helpful in the case of a repetitive URL that needs to be linked to.

\n

Unsafe links

\n

These are links that should not convert (source)

\n

[a](javascript:prompt(document.cookie))
\n[a](j a v a s c r i p t:prompt(document.cookie))
\n![a](javascript:prompt(document.cookie))
\n<javascript:prompt(document.cookie)>
\n<&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29>
\n![a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\na
\n![a'"`onerror=prompt(document.cookie)](x)
\n[citelol]: (javascript:prompt(document.cookie))
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[test](javascript://%0d%0aprompt(1))
\n[test](javascript://%0d%0aprompt(1);com)
\n[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
\n[notmalicious](javascript://%0d%0awindow.onerror=alert;throw%20document.cookie)
\n[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
\n[clickme](vbscript:alert(document.domain))
\nhttp://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
\ntext
\n[a](javascript:this;alert(1))
\n[a](javascript:this;alert(1))
\n[a](Javascript:alert(1))
\n[a](Javas%26%2399;ript:alert(1))
\n[a](javascript:alert�(1))
\n[a](javascript:confirm(1)
\n[a](javascript://www.google.com%0Aprompt(1))
\n[a](javascript://%0d%0aconfirm(1);com)
\n[a](javascript:window.onerror=confirm;throw%201)
\n[a](x01javascript:alert(document.domain))
\n[a](javascript://www.google.com%0Aalert(1))
\na
\n[a](JaVaScRiPt:alert(1))
\n\"a\"
\n\"a\"
\n</http://<?php><\\h1>script:scriptconfirm(2)
\nXSS
\n[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
\n[ ](http://a?p=[[/onclick=alert(0) .]])
\n[a](javascript:new%20Function`al\\ert`1``;)

" +basic = """

Links

+

Hasgeek
+Hasgeek TV

+

Autoconverted links

+

https://github.com/nodeca/pica
+http://twitter.com/hasgeek

+

Footnote style syntax

+

Hasgeek

+

Links can have a footnote style syntax, where the links can be defined later. This is helpful in the case of a repetitive URL that needs to be linked to.

+

Unsafe links

+

These are links that should not convert (source)

+

[a](javascript:prompt(document.cookie))
+[a](j a v a s c r i p t:prompt(document.cookie))
+![a](javascript:prompt(document.cookie))
+<javascript:prompt(document.cookie)>
+<&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29>
+![a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
+[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
+a
+![a'"`onerror=prompt(document.cookie)](x)
+[citelol]: (javascript:prompt(document.cookie))
+[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
+[test](javascript://%0d%0aprompt(1))
+[test](javascript://%0d%0aprompt(1);com)
+[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
+[notmalicious](javascript://%0d%0awindow.onerror=alert;throw%20document.cookie)
+[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
+[clickme](vbscript:alert(document.domain))
+http://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
+text
+[a](javascript:this;alert(1))
+[a](javascript:this;alert(1))
+[a](Javascript:alert(1))
+[a](Javas%26%2399;ript:alert(1))
+[a](javascript:alert�(1))
+[a](javascript:confirm(1)
+[a](javascript://www.google.com%0Aprompt(1))
+[a](javascript://%0d%0aconfirm(1);com)
+[a](javascript:window.onerror=confirm;throw%201)
+[a](x01javascript:alert(document.domain))
+[a](javascript://www.google.com%0Aalert(1))
+a
+[a](JaVaScRiPt:alert(1))
+a
+a
+</http://<?php><\\h1>script:scriptconfirm(2)
+XSS
+[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
+[ ](http://a?p=[[/onclick=alert(0) .]])
+[a](javascript:new%20Function`al\\ert`1``;)

+""" +document = """ +

Hasgeek
+Hasgeek TV

+ +

https://github.com/nodeca/pica
+http://twitter.com/hasgeek

+

Footnote style syntax

+

Hasgeek

+

Links can have a footnote style syntax, where the links can be defined later. This is helpful in the case of a repetitive URL that needs to be linked to.

+ +

These are links that should not convert (source)

+

[a](javascript:prompt(document.cookie))
+[a](j a v a s c r i p t:prompt(document.cookie))
+![a](javascript:prompt(document.cookie))
+<javascript:prompt(document.cookie)>
+<&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29>
+![a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
+[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
+a
+![a'"`onerror=prompt(document.cookie)](x)
+[citelol]: (javascript:prompt(document.cookie))
+[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
+[test](javascript://%0d%0aprompt(1))
+[test](javascript://%0d%0aprompt(1);com)
+[notmalicious](javascript:window.onerror=alert;throw%20document.cookie)
+[notmalicious](javascript://%0d%0awindow.onerror=alert;throw%20document.cookie)
+[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
+[clickme](vbscript:alert(document.domain))
+http://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
+text
+[a](javascript:this;alert(1))
+[a](javascript:this;alert(1))
+[a](Javascript:alert(1))
+[a](Javas%26%2399;ript:alert(1))
+[a](javascript:alert�(1))
+[a](javascript:confirm(1)
+[a](javascript://www.google.com%0Aprompt(1))
+[a](javascript://%0d%0aconfirm(1);com)
+[a](javascript:window.onerror=confirm;throw%201)
+[a](x01javascript:alert(document.domain))
+[a](javascript://www.google.com%0Aalert(1))
+a
+[a](JaVaScRiPt:alert(1))
+a
+a
+</http://<?php><\\h1>script:scriptconfirm(2)
+XSS
+[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
+[ ](http://a?p=[[/onclick=alert(0) .]])
+[a](javascript:new%20Function`al\\ert`1``;)

+""" diff --git a/tests/data/markdown/lists.toml b/tests/data/markdown/lists.toml index 057ef157b..53aee3185 100644 --- a/tests/data/markdown/lists.toml +++ b/tests/data/markdown/lists.toml @@ -36,5 +36,105 @@ Start numbering with offset: profiles = [ "basic", "document",] [expected_output] -basic = "

Lists

\n

Unordered

\n
    \n
  • Create a list by starting a line with +, -, or *
    • \n
  • Sub-lists are made by indenting 2 spaces:\n
      \n
    • Marker character change forces new list start:\n
        \n
      • Ac tristique libero volutpat at
        • \n
      \n
        \n
      • Facilisis in pretium nisl aliquet
        • \n
      \n
        \n
      • Nulla volutpat aliquam velit
        • \n
      \n
      • \n
    \n
    • \n
  • Very easy!
    • \n
\n

Ordered

\n
    \n
  1. \n

    Lorem ipsum dolor sit amet

    \n
    2. \n
  2. \n

    Consectetur adipiscing elit

    \n
    3. \n
  3. \n

    Integer molestie lorem at massa

    \n
    4. \n
  4. \n

    You can use sequential numbers...

    \n
    5. \n
  5. \n

    ...or keep all the numbers as 1.

    \n
    6. \n
\n
    \n
  1. You can start a new list by switching between ) and .
    2. \n
  2. Skipping numbers does not have any effect.
    3. \n
\n

Start numbering with offset:

\n
    \n
  1. foo
    2. \n
  2. bar
    3. \n
\n
\n

2022. If you do not want to render a list, use \\ to escape the .

" -document = "

Lists

\n

Unordered

\n
    \n
  • Create a list by starting a line with +, -, or *
    • \n
  • Sub-lists are made by indenting 2 spaces:\n
      \n
    • Marker character change forces new list start:\n
        \n
      • Ac tristique libero volutpat at
        • \n
      \n
        \n
      • Facilisis in pretium nisl aliquet
        • \n
      \n
        \n
      • Nulla volutpat aliquam velit
        • \n
      \n
      • \n
    \n
    • \n
  • Very easy!
    • \n
\n

Ordered

\n
    \n
  1. \n

    Lorem ipsum dolor sit amet

    \n
    2. \n
  2. \n

    Consectetur adipiscing elit

    \n
    3. \n
  3. \n

    Integer molestie lorem at massa

    \n
    4. \n
  4. \n

    You can use sequential numbers...

    \n
    5. \n
  5. \n

    ...or keep all the numbers as 1.

    \n
    6. \n
\n
    \n
  1. You can start a new list by switching between ) and .
    2. \n
  2. Skipping numbers does not have any effect.
    3. \n
\n

Start numbering with offset:

\n
    \n
  1. foo
    2. \n
  2. bar
    3. \n
\n
\n

2022. If you do not want to render a list, use \\ to escape the .

" +basic = """

Lists

+

Unordered

+
    +
  • Create a list by starting a line with +, -, or *
    • +
  • Sub-lists are made by indenting 2 spaces: +
      +
    • Marker character change forces new list start: +
        +
      • Ac tristique libero volutpat at
        • +
      +
        +
      • Facilisis in pretium nisl aliquet
        • +
      +
        +
      • Nulla volutpat aliquam velit
        • +
      +
      • +
    +
    • +
  • Very easy!
    • +
+

Ordered

+
    +
  1. +

    Lorem ipsum dolor sit amet

    +
    2. +
  2. +

    Consectetur adipiscing elit

    +
    3. +
  3. +

    Integer molestie lorem at massa

    +
    4. +
  4. +

    You can use sequential numbers...

    +
    5. +
  5. +

    ...or keep all the numbers as 1.

    +
    6. +
+
    +
  1. You can start a new list by switching between ) and .
    2. +
  2. Skipping numbers does not have any effect.
    3. +
+

Start numbering with offset:

+
    +
  1. foo
    2. +
  2. bar
    3. +
+
+

2022. If you do not want to render a list, use \\ to escape the .

+""" +document = """

Lists

+

Unordered

+
    +
  • Create a list by starting a line with +, -, or *
    • +
  • Sub-lists are made by indenting 2 spaces: +
      +
    • Marker character change forces new list start: +
        +
      • Ac tristique libero volutpat at
        • +
      +
        +
      • Facilisis in pretium nisl aliquet
        • +
      +
        +
      • Nulla volutpat aliquam velit
        • +
      +
      • +
    +
    • +
  • Very easy!
    • +
+

Ordered

+
    +
  1. +

    Lorem ipsum dolor sit amet

    +
    2. +
  2. +

    Consectetur adipiscing elit

    +
    3. +
  3. +

    Integer molestie lorem at massa

    +
    4. +
  4. +

    You can use sequential numbers...

    +
    5. +
  5. +

    ...or keep all the numbers as 1.

    +
    6. +
+
    +
  1. You can start a new list by switching between ) and .
    2. +
  2. Skipping numbers does not have any effect.
    3. +
+

Start numbering with offset:

+
    +
  1. foo
    2. +
  2. bar
    3. +
+
+

2022. If you do not want to render a list, use \\ to escape the .

+""" diff --git a/tests/data/markdown/tables.toml b/tests/data/markdown/tables.toml index 3e5f7db98..52eace6b8 100644 --- a/tests/data/markdown/tables.toml +++ b/tests/data/markdown/tables.toml @@ -20,5 +20,63 @@ markdown = """ profiles = [ "basic", "document",] [expected_output] -basic = "

Tables

\n

| Option | Description |
\n| ------ | ----------- |
\n| data | path to data files to supply the data that will be passed into templates. |
\n| engine | engine to be used for processing templates. Handlebars is the default. |
\n| ext | extension to be used for dest files. |

\n

Right aligned columns

\n

| Option | Description |
\n| ------:| -----------:|
\n| data | path to data files to supply the data that will be passed into templates. |
\n| engine | engine to be used for processing templates. Handlebars is the default. |
\n| ext | extension to be used for dest files. |

" -document = "

Tables

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
\n

Right aligned columns

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
" +basic = """

Tables

+

| Option | Description |
+| ------ | ----------- |
+| data | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext | extension to be used for dest files. |

+

Right aligned columns

+

| Option | Description |
+| ------:| -----------:|
+| data | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext | extension to be used for dest files. |

+""" +document = """

Tables

+ + + + + + + + + + + + + + + + + + + + + +
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
+

Right aligned columns

+ + + + + + + + + + + + + + + + + + + + + +
OptionDescription
datapath to data files to supply the data that will be passed into templates.
engineengine to be used for processing templates. Handlebars is the default.
extextension to be used for dest files.
+""" diff --git a/tests/unit/utils/test_markdown.py b/tests/unit/utils/test_markdown.py index 34247be6e..c020f10ed 100644 --- a/tests/unit/utils/test_markdown.py +++ b/tests/unit/utils/test_markdown.py @@ -1,7 +1,7 @@ """Tests for markdown parser.""" from pathlib import Path -from typing import Dict, List, Optional, Type +from typing import Dict, List, Optional, Type, Union import warnings import pytest @@ -11,7 +11,7 @@ from funnel.utils.markdown.profiles import MarkdownProfile, profiles DATAROOT: Path = Path('tests/data/markdown') -DEBUG = True +DEBUG = False class MarkdownCase: @@ -69,38 +69,37 @@ def case_id(self) -> str: return f'{self.test_id}-{self.profile_id}' @property - def markdown_profile(self): + def markdown_profile(self) -> Union[str, Type[MarkdownProfile]]: return self.profile if self.profile is not None else self.profile_id @property - def output(self): - return ( - markdown(self.markdown, self.markdown_profile) - .__str__() - .lstrip('\n\r') - .rstrip(' \n\r') - ) + def output(self) -> str: + return markdown(self.markdown, self.markdown_profile) + + def update_expected_output(self) -> None: + self.expected_output = self.output class MarkdownTestRegistry: test_map: Optional[Dict[str, Dict[str, MarkdownCase]]] = None + test_files: Dict[str, tomlkit.TOMLDocument] @classmethod - def load(cls): + def load(cls) -> None: if cls.test_map is None: cls.test_map = {} - tests = { + cls.test_files = { file.name: tomlkit.loads(file.read_text()) for file in DATAROOT.iterdir() if file.suffix == '.toml' } - for test_id, test in tests.items(): - config = test['config'] - exp = test.get('expected_output', {}) + for test_id, test_data in cls.test_files.items(): + config = test_data['config'] + exp = test_data.get('expected_output', {}) cls.test_map[test_id] = { profile_id: MarkdownCase( test_id, - test['markdown'], + test_data['markdown'], profile_id, profile=profile, expected_output=exp.get(profile_id, None), @@ -111,6 +110,16 @@ def load(cls): }.items() } + @classmethod + def dump(cls) -> None: + if cls.test_map is not None: + for test_id, data in cls.test_files.items(): + data['expected_output'] = { + profile_id: tomlkit.api.string(case.output, multiline=True) + for profile_id, case in cls.test_map[test_id].items() + } + (DATAROOT / test_id).write_text(tomlkit.dumps(data)) + @classmethod def dataset(cls) -> List[MarkdownCase]: cls.load() @@ -120,6 +129,11 @@ def dataset(cls) -> List[MarkdownCase]: else [] ) + @classmethod + def update_output(cls) -> None: + cls.load() + cls.dump() + @pytest.mark.parametrize( 'case', @@ -134,3 +148,12 @@ def test_markdown_cases(case: MarkdownCase, unified_diff_output) -> None: unified_diff_output(case.expected_output, case.output) else: assert case.expected_output == case.output + + +@pytest.mark.update_markdown_data() +def test_markdown_update_output(pytestconfig): + has_mark = pytestconfig.getoption('-m', default=None) == 'update_markdown_data' + if not has_mark: + pytest.skip('Skipping update of expected output of markdown test cases') + # update_markdown_tests_data(debug=False) + MarkdownTestRegistry.update_output() From 740ea5cbec09047904f665665497a601f765426d Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 3 Nov 2022 20:32:55 +0530 Subject: [PATCH 25/72] Generate markdown debug output.html file. Cleanup. #1490 Also small changes in profile structure for #1480 --- Makefile | 7 ++- funnel/utils/markdown/base.py | 9 ++-- funnel/views/api/markdown.py | 8 +++- pyproject.toml | 3 +- tests/unit/utils/test_markdown.py | 79 ++++++++++++++++++++++++++----- 5 files changed, 86 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 5d2357021..8c4173008 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,10 @@ deps-dev: deps-test: pip-compile --upgrade requirements_test.in -tests-data: tests-data-md +tests-data: tests-data-markdown -tests-data-md: +tests-data-markdown: pytest -v -m update_markdown_data + +debug-markdown-tests: + pytest -v -m debug_markdown_output diff --git a/funnel/utils/markdown/base.py b/funnel/utils/markdown/base.py index 7d7efdf83..3115f95d4 100644 --- a/funnel/utils/markdown/base.py +++ b/funnel/utils/markdown/base.py @@ -33,17 +33,17 @@ @overload -def markdown(text: None, profile: Optional[Union[str, Type[MarkdownProfile]]]) -> None: +def markdown(text: None, profile: Union[str, Type[MarkdownProfile]]) -> None: ... @overload -def markdown(text: str, profile: Optional[Union[str, Type[MarkdownProfile]]]) -> Markup: +def markdown(text: str, profile: Union[str, Type[MarkdownProfile]]) -> Markup: ... def markdown( - text: Optional[str], profile: Optional[Union[str, Type[MarkdownProfile]]] + text: Optional[str], profile: Union[str, Type[MarkdownProfile]] ) -> Optional[Markup]: """ Markdown parser compliant with Commonmark+GFM using markdown-it-py. @@ -56,7 +56,7 @@ def markdown( # Replace invisible characters with spaces text = normalize_spaces_multiline(text) - if profile is None or isinstance(profile, str): + if isinstance(profile, str): try: _profile = profiles[profile] except KeyError as exc: @@ -70,6 +70,7 @@ def markdown( 'Wrong type - profile has to be either str or a subclass of MarkdownProfile' ) + # TODO: Move MarkdownIt instance generation to profile class method md = MarkdownIt(*_profile.args) if md.linkify is not None: diff --git a/funnel/views/api/markdown.py b/funnel/views/api/markdown.py index 490553de5..0ef9be5f7 100644 --- a/funnel/views/api/markdown.py +++ b/funnel/views/api/markdown.py @@ -15,8 +15,12 @@ def markdown_preview() -> ReturnView: """Render Markdown in the backend, with custom options based on use case.""" profile: Optional[str] = request.form.get('profile') - if profile not in profiles: - profile = None + if profile is None or profile not in profiles: + return { + 'status': 'error', + 'error': 'not_implemented', + 'error_description': f'Markdown profile {profile} is not supported', + }, 501 text = request.form.get('text') html = markdown(text, profile) diff --git a/pyproject.toml b/pyproject.toml index 41fa67f0b..0979be103 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,7 +123,8 @@ markers = [ 'requires_config("feature"): Run test only if specified app config is available', 'formdata(dict): HTTP form data for form fixture', 'formuser("user"): User fixture for editing a form', - 'update_markdown_data: Regenerate markdown test output (dev use only)' + 'update_markdown_data: Regenerate markdown test output (dev use only)', + 'debug_markdown_output: Generate markdown debug file tests/data/markdown/output.html (dev use only)' ] filterwarnings = ['ignore::DeprecationWarning', 'ignore::UserWarning'] mocked-sessions = 'funnel.db.db.session' # For pytest-flask-sqlalchemy (unused) diff --git a/tests/unit/utils/test_markdown.py b/tests/unit/utils/test_markdown.py index c020f10ed..054b4e3e9 100644 --- a/tests/unit/utils/test_markdown.py +++ b/tests/unit/utils/test_markdown.py @@ -1,9 +1,13 @@ """Tests for markdown parser.""" +from copy import copy +from datetime import datetime from pathlib import Path from typing import Dict, List, Optional, Type, Union import warnings +from bs4 import BeautifulSoup +from markupsafe import Markup import pytest import tomlkit @@ -11,7 +15,6 @@ from funnel.utils.markdown.profiles import MarkdownProfile, profiles DATAROOT: Path = Path('tests/data/markdown') -DEBUG = False class MarkdownCase: @@ -121,7 +124,7 @@ def dump(cls) -> None: (DATAROOT / test_id).write_text(tomlkit.dumps(data)) @classmethod - def dataset(cls) -> List[MarkdownCase]: + def test_cases(cls) -> List[MarkdownCase]: cls.load() return ( [case for tests in cls.test_map.values() for case in tests.values()] @@ -130,24 +133,71 @@ def dataset(cls) -> List[MarkdownCase]: ) @classmethod - def update_output(cls) -> None: + def update_expected_output(cls) -> None: cls.load() cls.dump() + @classmethod + def update_debug_output(cls) -> None: + cls.load() + template = BeautifulSoup( + (DATAROOT / 'template.html').read_text(), 'html.parser' + ) + case_template = template.find(id='output_template') + for case in cls.test_cases(): + op = copy(case_template) + del op['id'] + op.select('.filename')[0].string = case.test_id + op.select('.profile')[0].string = str(case.profile_id) + op.select('.config')[0].string = '' + op.select('.markdown .output')[0].append(case.markdown) + op.select('.expected .output')[0].append( + BeautifulSoup(case.expected_output, 'html.parser') + if case.expected_output is not None + else 'Not generated' + ) + op.select('.final_output .output')[0].append( + BeautifulSoup(case.output, 'html.parser') + ) + op['class'] = op.get('class', []) + [ + 'success' if case.expected_output == case.output else 'failed' + ] + template.find('body').append(op) + template.find(id='generated').string = datetime.now().strftime( + '%d %B, %Y %H:%M:%S' + ) + (DATAROOT / 'output.html').write_text(template.prettify()) + + +def test_markdown_none() -> None: + assert markdown(None, 'basic') is None + assert markdown(None, 'document') is None + assert markdown(None, 'text-field') is None + assert markdown(None, MarkdownProfile) is None + + +def test_markdown_blank() -> None: + blank_response = Markup('') + assert markdown('', 'basic') == blank_response + assert markdown('', 'document') == blank_response + assert markdown('', 'text-field') == blank_response + assert markdown('', MarkdownProfile) == blank_response + @pytest.mark.parametrize( 'case', - MarkdownTestRegistry.dataset(), + MarkdownTestRegistry.test_cases(), ) -def test_markdown_cases(case: MarkdownCase, unified_diff_output) -> None: +# def test_markdown_cases(case: MarkdownCase, unified_diff_output) -> None: +def test_markdown_cases(case: MarkdownCase) -> None: if case.expected_output is None: warnings.warn(f'Expected output not generated for {case}') pytest.skip(f'Expected output not generated for {case}') - if DEBUG: - unified_diff_output(case.expected_output, case.output) - else: - assert case.expected_output == case.output + assert case.expected_output == case.output + + # Debug function + # unified_diff_output(case.expected_output, case.output) @pytest.mark.update_markdown_data() @@ -155,5 +205,12 @@ def test_markdown_update_output(pytestconfig): has_mark = pytestconfig.getoption('-m', default=None) == 'update_markdown_data' if not has_mark: pytest.skip('Skipping update of expected output of markdown test cases') - # update_markdown_tests_data(debug=False) - MarkdownTestRegistry.update_output() + MarkdownTestRegistry.update_expected_output() + + +@pytest.mark.debug_markdown_output() +def test_markdown_debug_output(pytestconfig): + has_mark = pytestconfig.getoption('-m', default=None) == 'debug_markdown_output' + if not has_mark: + pytest.skip('Skipping update of debug output file for markdown test cases') + MarkdownTestRegistry.update_debug_output() From 756e882f6fe7f244ffe81ebaad4c9f424684cc84 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 4 Nov 2022 16:59:16 +0530 Subject: [PATCH 26/72] Commit missed in 740ea5cbec09047904f665665497a601f765426d Related to #1480 --- funnel/utils/markdown/profiles.py | 1 - 1 file changed, 1 deletion(-) diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 9cf7d7e72..ba8171639 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -71,7 +71,6 @@ class MarkdownProfileTextField(MarkdownProfile): profiles: Dict[Optional[str], Type[MarkdownProfile]] = { - None: MarkdownProfileDocument, 'basic': MarkdownProfileBasic, 'document': MarkdownProfileDocument, 'text-field': MarkdownProfileTextField, From c7f9bcad69b20f1d01fc6ce6df093118d9e6bee7 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 7 Nov 2022 18:48:23 +0530 Subject: [PATCH 27/72] Updated expected_output for the test basic.html to accommodate for change of output for ~text~ from using to #1493 --- tests/data/markdown/basic.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/data/markdown/basic.toml b/tests/data/markdown/basic.toml index 78ff9d637..01a722dc6 100644 --- a/tests/data/markdown/basic.toml +++ b/tests/data/markdown/basic.toml @@ -140,7 +140,7 @@ test.. test... test..... test?..... test!....
""" document = """

Basic markup

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

-

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
+

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
It also has a newline break here!!!

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

__Bold without closure does not work
@@ -150,7 +150,7 @@ _Emphasis without closure does not work

Bold without closure
on the same line
carries forward to
-this is text that should strike off
+this is text that should strike off
multiple consecutive lines

Bold without closure
on the same line
@@ -167,7 +167,7 @@ multiple consecutive lines

Horizontal Rules

Above is a horizontal rule using hyphens.
-This is text that should strike off

+This is text that should strike off

Above is a horizontal rule using asterisks.
Below is a horizontal rule using underscores.

From c9a84b32f37d716325c1a2a504e7fd36e404e840 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 7 Nov 2022 19:23:23 +0530 Subject: [PATCH 28/72] Tests for markdown-it-py plugins for ins/del/sup/sub/mark #1493 --- tests/data/markdown/basic_plugins.toml | 324 ++++++++++++++++++++++ tests/data/markdown/ext_ins.toml.disabled | 92 ------ 2 files changed, 324 insertions(+), 92 deletions(-) create mode 100644 tests/data/markdown/basic_plugins.toml delete mode 100644 tests/data/markdown/ext_ins.toml.disabled diff --git a/tests/data/markdown/basic_plugins.toml b/tests/data/markdown/basic_plugins.toml new file mode 100644 index 000000000..33ed6b3ae --- /dev/null +++ b/tests/data/markdown/basic_plugins.toml @@ -0,0 +1,324 @@ +markdown = """ +## Testing markdown-it-py plugins for ins/del/sup/sub/mark + +This is ++inserted text++~~and deleted text~~. + +This is ^superscript^, this is ~subscript~ and this is ==marked text==. + +This is not +inserted text+, nor is this ~deleted text~. + +^Super script^ and ~sub script~, do not support whitespaces. + +++This is a +multiline insert++ ~~and a + multiline delete~~ + +~~~html +
+    
+        This should not render as deleted text, rather it should be a fenced block.
+    
+
+~~~ + +This is a paragraph. +++This line has been inserted in it.++ +This is the middle of the paragraph. +++These lines have +also been inserted in it.++ +~~While this line has been deleted.~~ +The paragraph ends here. + +This is +++an insert\\++ed text+++ +This is ++++also an insert\\++ed text++++ + +This text has ++an inserted text ++with another inserted text++++ + +This text has ++an inserted text ++with another inserted text++ ++with another inserted text++++ + ++Hello World! I cannot assert myself as an inserted text.+ +++`foobar()`++ + +This is ~~~a delet\\~~ed text.~~~ +This is ~~~~also a delet\\~~ed text~~~~ + +This text has ~~a deleted text ~~with another deleted text~~~~ + +This text has ~~a deleted text ~~with another deleted text~~ ~~with another inserted text~~~~ + +~Hello World! I cannot assert myself as an inserted text nor a subscript.~ +~~`foobar()`~~ + +Now, we will list out some ~~fruits~~++grocery++: +- ++Fruits++ + + Apples + + ++Grapes++ + + Pears +- ++Vegetables++ + + ++Potato++ + + ++Tomato++ + +**bold++insert++** +**bold ++_insert emphasized_++** +**bold _++emphasized insert++_** +**bold++_insert emphasized_++** +**bold_++emphasized insert++_** + +__bold~~delete~~__ +__bold ~~*delete emphasized*~~__ +__bold *~~emphasized delete~~*__ +__bold~~*delete emphasized*~~__ +__bold*~~emphasized delete~~*__ + +__bold^super^__ +__bold ^*super-emphasized*^__ +__bold *^emphasized-super^*__ +__bold^*super-emphasized*^__ +__bold*^emphasized-super^*__ + +__bold~sub~__ +__bold ~*sub-emphasized*~__ +__bold *~emphasized-sub~*__ +__bold~*sub-emphasized*~__ +__bold*~emphasized-sub~*__ + +__bold==mark==__ +__bold ==*mark emphasized*==__ +__bold *==emphasized mark==*__ +__bold==*mark emphasized*==__ +__bold*==emphasized mark==*__ +""" + +[config] +profiles = [ "basic", "document",] + +[config.custom_profiles.basic_plugins] +args_config = "default" +plugins = [ "ins","del","sup","sub","mark"] + +[expected_output] +basic = """

Testing markdown-it-py plugins for ins/del/sup/sub/mark

+

This is ++inserted text++and deleted text.

+

This is ^superscript^, this is ~subscript~ and this is ==marked text==.

+

This is not +inserted text+, nor is this ~deleted text~.

+

^Super script^ and ~sub script~, do not support whitespaces.

+

++This is a
+multiline insert++ and a
+multiline delete

+
<pre>
+    <code>
+        This should not render as deleted text, rather it should be a fenced block.
+    </code>
+</pre>
+
+

This is a paragraph.
+++This line has been inserted in it.++
+This is the middle of the paragraph.
+++These lines have
+also been inserted in it.++
+While this line has been deleted.
+The paragraph ends here.

+

This is +++an insert++ed text+++
+This is ++++also an insert++ed text++++

+

This text has ++an inserted text ++with another inserted text++++

+

This text has ++an inserted text ++with another inserted text++ ++with another inserted text++++

+

+Hello World! I cannot assert myself as an inserted text.+
+++foobar()++

+

This is ~a delet~~ed text.~
+This is also a delet~~ed text

+

This text has a deleted text with another deleted text

+

This text has a deleted text with another deleted text with another inserted text

+

~Hello World! I cannot assert myself as an inserted text nor a subscript.~
+foobar()

+

Now, we will list out some fruits++grocery++:

+
    +
  • ++Fruits++ +
      +
    • Apples
      • +
    • ++Grapes++
      • +
    • Pears
      • +
    +
    • +
  • ++Vegetables++ +
      +
    • ++Potato++
      • +
    • ++Tomato++
      • +
    +
    • +
+

bold++insert++
+bold ++insert emphasized++
+bold ++emphasized insert++
+bold++insert emphasized++
+bold_++emphasized insert++_

+

bolddelete
+bold delete emphasized
+bold emphasized delete
+bold~~delete emphasized~~
+bold*emphasized delete*

+

bold^super^
+bold ^super-emphasized^
+bold ^emphasized-super^
+bold^super-emphasized^
+bold*^emphasized-super^*

+

bold~sub~
+bold ~sub-emphasized~
+bold ~emphasized-sub~
+bold~sub-emphasized~
+bold*~emphasized-sub~*

+

bold==mark==
+bold ==mark emphasized==
+bold ==emphasized mark==
+bold==mark emphasized==
+bold*==emphasized mark==*

+""" +document = """

Testing markdown-it-py plugins for ins/del/sup/sub/mark

+

This is inserted textand deleted text.

+

This is superscript, this is subscript and this is marked text.

+

This is not +inserted text+, nor is this ~deleted text~.

+

^Super script^ and ~sub script~, do not support whitespaces.

+

This is a
+multiline insert and a
+multiline delete

+
<pre>
+    <code>
+        This should not render as deleted text, rather it should be a fenced block.
+    </code>
+</pre>
+
+

This is a paragraph.
+This line has been inserted in it.
+This is the middle of the paragraph.
+These lines have
+also been inserted in it.
+While this line has been deleted.
+The paragraph ends here.

+

This is +an insert++ed text+
+This is also an insert++ed text

+

This text has an inserted text with another inserted text

+

This text has an inserted text with another inserted text with another inserted text

+

+Hello World! I cannot assert myself as an inserted text.+
+foobar()

+

This is ~a delet~~ed text.~
+This is also a delet~~ed text

+

This text has a deleted text with another deleted text

+

This text has a deleted text with another deleted text with another inserted text

+

~Hello World! I cannot assert myself as an inserted text nor a subscript.~
+foobar()

+

Now, we will list out some fruitsgrocery:

+
    +
  • Fruits +
      +
    • Apples
      • +
    • Grapes
      • +
    • Pears
      • +
    +
    • +
  • Vegetables +
      +
    • Potato
      • +
    • Tomato
      • +
    +
    • +
+

boldinsert
+bold insert emphasized
+bold emphasized insert
+bold++insert emphasized++
+bold_emphasized insert_

+

bolddelete
+bold delete emphasized
+bold emphasized delete
+bold~~delete emphasized~~
+bold*emphasized delete*

+

boldsuper
+bold *super-emphasized*
+bold emphasized-super
+bold*super-emphasized*
+bold*emphasized-super*

+

boldsub
+bold *sub-emphasized*
+bold emphasized-sub
+bold*sub-emphasized*
+bold*emphasized-sub*

+

boldmark
+bold mark emphasized
+bold emphasized mark
+bold==mark emphasized==
+bold*emphasized mark*

+""" +basic_plugins = """

Testing markdown-it-py plugins for ins/del/sup/sub/mark

+

This is inserted textand deleted text.

+

This is superscript, this is subscript and this is marked text.

+

This is not +inserted text+, nor is this ~deleted text~.

+

^Super script^ and ~sub script~, do not support whitespaces.

+

This is a
+multiline insert and a
+multiline delete

+
<pre>
+    <code>
+        This should not render as deleted text, rather it should be a fenced block.
+    </code>
+</pre>
+
+

This is a paragraph.
+This line has been inserted in it.
+This is the middle of the paragraph.
+These lines have
+also been inserted in it.
+While this line has been deleted.
+The paragraph ends here.

+

This is +an insert++ed text+
+This is also an insert++ed text

+

This text has an inserted text with another inserted text

+

This text has an inserted text with another inserted text with another inserted text

+

+Hello World! I cannot assert myself as an inserted text.+
+foobar()

+

This is ~a delet~~ed text.~
+This is also a delet~~ed text

+

This text has a deleted text with another deleted text

+

This text has a deleted text with another deleted text with another inserted text

+

~Hello World! I cannot assert myself as an inserted text nor a subscript.~
+foobar()

+

Now, we will list out some fruitsgrocery:

+
    +
  • Fruits +
      +
    • Apples
      • +
    • Grapes
      • +
    • Pears
      • +
    +
    • +
  • Vegetables +
      +
    • Potato
      • +
    • Tomato
      • +
    +
    • +
+

boldinsert
+bold insert emphasized
+bold emphasized insert
+bold++insert emphasized++
+bold_emphasized insert_

+

bolddelete
+bold delete emphasized
+bold emphasized delete
+bold~~delete emphasized~~
+bold*emphasized delete*

+

boldsuper
+bold *super-emphasized*
+bold emphasized-super
+bold*super-emphasized*
+bold*emphasized-super*

+

boldsub
+bold *sub-emphasized*
+bold emphasized-sub
+bold*sub-emphasized*
+bold*emphasized-sub*

+

boldmark
+bold mark emphasized
+bold emphasized mark
+bold==mark emphasized==
+bold*emphasized mark*

+""" diff --git a/tests/data/markdown/ext_ins.toml.disabled b/tests/data/markdown/ext_ins.toml.disabled deleted file mode 100644 index 92582ab99..000000000 --- a/tests/data/markdown/ext_ins.toml.disabled +++ /dev/null @@ -1,92 +0,0 @@ -[data] -markdown = """ -## Extension/plugin for `` -This is a paragraph. -++This line has been inserted in it.++ -This is the middle of the paragraph. -++These lines have -also been inserted in it++ -The paragraph ends here. - -This is +++an insert\\+ed text+++ -This is ++++also an insert\\+ed text++++ - -This text has ++an inserted text ++with another inserted text++++ - -This text has ++an inserted text ++with another inserted text++ ++with another inserted text++++ - -+Hello World! I cannot assert myself as an inserted text.+ -++`foobar()`++ - -Now, we will list out some ~~fruits~~++grocery++: -- ++Fruits++ - + Apples - + ++Grapes++ - + Pears -- ++Vegetables++ - + ++Potato++ - + ++Tomato++ - -**bold++insert++** -**bold ++_insert emphasized_++** -**bold _++emphasized insert++_** -**bold++_insert emphasized_++** -**bold_++emphasized insert++_** - -__bold++insert++__ -__bold ++*insert emphasized*++__ -__bold *++emphasized insert++*__ -__bold++*insert emphasized*++__ -__bold*++emphasized insert++*__ - -*emphasized++insert++* -*emphasized ++__insert bold__++* -*emphasized __++bold insert++__* -*emphasized++__insert bold__++* -*emphasized__++bold insert++__* - -_emphasized++insert++_ -_emphasized ++**insert bold**++_ -_emphasized **++bold insert++**_ -_emphasized++**insert bold**++_ -_emphasized**++bold insert++**_ - -**bold -text++insert++ -text ++insert++ -text_emphasized_ -text _emphasized_ -end** - -__bold -text++insert++ -text ++insert++ -text*emphasized* -text *emphasized* -end__ - -*emphasized -text++insert++ -text ++insert++ -text__bold__ -text __bold__ -end* - -_emphasized -text++insert++ -text ++insert++ -text**bold** -text **bold** -end_ -""" - -[config] -profiles = [ "basic", "document",] - -[config.extra_configs.ins] -extensions = [ "ins",] - -[expected_output] -"" = "

Extension/plugin for <ins>

\n

This is a paragraph.
\n++This line has been inserted in it.++
\nThis is the middle of the paragraph.
\n++These lines have
\nalso been inserted in it++
\nThe paragraph ends here.

\n

This is +++an insert+ed text+++
\nThis is ++++also an insert+ed text++++

\n

This text has ++an inserted text ++with another inserted text++++

\n

This text has ++an inserted text ++with another inserted text++ ++with another inserted text++++

\n

+Hello World! I cannot assert myself as an inserted text.+
\n++foobar()++

\n

Now, we will list out some fruits++grocery++:

\n
    \n
  • ++Fruits++\n
      \n
    • Apples
      • \n
    • ++Grapes++
      • \n
    • Pears
      • \n
    \n
    • \n
  • ++Vegetables++\n
      \n
    • ++Potato++
      • \n
    • ++Tomato++
      • \n
    \n
    • \n
\n

bold++insert++
\nbold ++insert emphasized++
\nbold ++emphasized insert++
\nbold++insert emphasized++
\nbold_++emphasized insert++_

\n

bold++insert++
\nbold ++insert emphasized++
\nbold ++emphasized insert++
\nbold++insert emphasized++
\nbold*++emphasized insert++*

\n

emphasized++insert++
\nemphasized ++insert bold++
\nemphasized ++bold insert++
\nemphasized++insert bold++
\nemphasized__++bold insert++__

\n

emphasized++insert++
\nemphasized ++insert bold++
\nemphasized ++bold insert++
\nemphasized++insert bold++
\nemphasized**++bold insert++**

\n

bold
\ntext++insert++
\ntext ++insert++
\ntext_emphasized_
\ntext emphasized
\nend

\n

bold
\ntext++insert++
\ntext ++insert++
\ntextemphasized
\ntext emphasized
\nend

\n

emphasized
\ntext++insert++
\ntext ++insert++
\ntext__bold__
\ntext bold
\nend

\n

emphasized
\ntext++insert++
\ntext ++insert++
\ntextbold
\ntext bold
\nend

" -no_ext = "

Extension/plugin for <ins>

\n

This is a paragraph.
\n++This line has been inserted in it.++
\nThis is the middle of the paragraph.
\n++These lines have
\nalso been inserted in it++
\nThe paragraph ends here.

\n

This is +++an insert+ed text+++
\nThis is ++++also an insert+ed text++++

\n

This text has ++an inserted text ++with another inserted text++++

\n

This text has ++an inserted text ++with another inserted text++ ++with another inserted text++++

\n

+Hello World! I cannot assert myself as an inserted text.+
\n++foobar()++

\n

Now, we will list out some fruits++grocery++:

\n
    \n
  • ++Fruits++\n
      \n
    • Apples
      • \n
    • ++Grapes++
      • \n
    • Pears
      • \n
    \n
    • \n
  • ++Vegetables++\n
      \n
    • ++Potato++
      • \n
    • ++Tomato++
      • \n
    \n
    • \n
\n

bold++insert++
\nbold ++insert emphasized++
\nbold ++emphasized insert++
\nbold++insert emphasized++
\nbold_++emphasized insert++_

\n

bold++insert++
\nbold ++insert emphasized++
\nbold ++emphasized insert++
\nbold++insert emphasized++
\nbold*++emphasized insert++*

\n

emphasized++insert++
\nemphasized ++insert bold++
\nemphasized ++bold insert++
\nemphasized++insert bold++
\nemphasized__++bold insert++__

\n

emphasized++insert++
\nemphasized ++insert bold++
\nemphasized ++bold insert++
\nemphasized++insert bold++
\nemphasized**++bold insert++**

\n

bold
\ntext++insert++
\ntext ++insert++
\ntext_emphasized_
\ntext emphasized
\nend

\n

bold
\ntext++insert++
\ntext ++insert++
\ntextemphasized
\ntext emphasized
\nend

\n

emphasized
\ntext++insert++
\ntext ++insert++
\ntext__bold__
\ntext bold
\nend

\n

emphasized
\ntext++insert++
\ntext ++insert++
\ntextbold
\ntext bold
\nend

" -ins = "

Extension/plugin for <ins>

\n

This is a paragraph.
\n++This line has been inserted in it.++
\nThis is the middle of the paragraph.
\n++These lines have
\nalso been inserted in it++
\nThe paragraph ends here.

\n

This is +++an insert+ed text+++
\nThis is ++++also an insert+ed text++++

\n

This text has ++an inserted text ++with another inserted text++++

\n

This text has ++an inserted text ++with another inserted text++ ++with another inserted text++++

\n

+Hello World! I cannot assert myself as an inserted text.+
\n++foobar()++

\n

Now, we will list out some fruits++grocery++:

\n
    \n
  • ++Fruits++\n
      \n
    • Apples
      • \n
    • ++Grapes++
      • \n
    • Pears
      • \n
    \n
    • \n
  • ++Vegetables++\n
      \n
    • ++Potato++
      • \n
    • ++Tomato++
      • \n
    \n
    • \n
\n

bold++insert++
\nbold ++insert emphasized++
\nbold ++emphasized insert++
\nbold++insert emphasized++
\nbold_++emphasized insert++_

\n

bold++insert++
\nbold ++insert emphasized++
\nbold ++emphasized insert++
\nbold++insert emphasized++
\nbold*++emphasized insert++*

\n

emphasized++insert++
\nemphasized ++insert bold++
\nemphasized ++bold insert++
\nemphasized++insert bold++
\nemphasized__++bold insert++__

\n

emphasized++insert++
\nemphasized ++insert bold++
\nemphasized ++bold insert++
\nemphasized++insert bold++
\nemphasized**++bold insert++**

\n

bold
\ntext++insert++
\ntext ++insert++
\ntext_emphasized_
\ntext emphasized
\nend

\n

bold
\ntext++insert++
\ntext ++insert++
\ntextemphasized
\ntext emphasized
\nend

\n

emphasized
\ntext++insert++
\ntext ++insert++
\ntext__bold__
\ntext bold
\nend

\n

emphasized
\ntext++insert++
\ntext ++insert++
\ntextbold
\ntext bold
\nend

" From 09a22ec757591332e58b7d86b9b377b684c9e346 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 8 Nov 2022 18:45:01 +0530 Subject: [PATCH 29/72] Tests for embeddable fenced blocks for markmap, vega-lite and mermaid #1496 --- tests/data/markdown/markmap.toml | 403 ++++++++ tests/data/markdown/mermaid.toml | 450 +++++++++ tests/data/markdown/vega-lite.toml | 1369 ++++++++++++++++++++++++++++ 3 files changed, 2222 insertions(+) create mode 100644 tests/data/markdown/markmap.toml create mode 100644 tests/data/markdown/mermaid.toml create mode 100644 tests/data/markdown/vega-lite.toml diff --git a/tests/data/markdown/markmap.toml b/tests/data/markdown/markmap.toml new file mode 100644 index 000000000..72c7482d1 --- /dev/null +++ b/tests/data/markdown/markmap.toml @@ -0,0 +1,403 @@ +markdown = """ +### Markmap + +```{markmap} + +# Digital Identifiers and Rights + +## Community and Outreach + +- Experts +- Grant organizations +- IT ministries +- Journalism schools +- Journalists and reporters +- Partnerships + - Patrons + - Sponsors +- Policymakers +- Rights activists +- Startups +- Thinktanks +- Venture capital +- Volunteers + +## Domains + +- Border controls +- Citizenship +- Digital data trusts +- FinTech +- Government +- Health services delivery +- Hospitality +- Law enforcement +- Online retail and commerce +- Smart automation +- Social media +- Travel and tourism + +## Location + +- International +- Local or domestic +- Transit + +## Output and Outcomes + +- Best practices guide for product/service development +- Conference +- Conversations (eg. Twitter Spaces) +- Masterclass webinars +- Proceedings (talk playlist) +- Reports +- Review of Policies + +## Themes + +### Digital Identity + +- Anonymity +- Architecture of digital trust +- Control and ownership +- Identity and identifier models +- Inclusion and exclusion +- Portability +- Principles +- Regulations +- Reputation +- Rights and agency +- Trust framework +- Verifiability +- Vulnerable communities + +### Digital Rights + +- Current state across region +- Harms +- Emerging regulatory requirements +- Web 3.0 and decentralization +- Naturalization + +## Streams + +- Banking and finance +- Data exchange and interoperability +- Data governance models +- Data markets +- Digital identifiers and identity systems +- Digital public goods +- Digital public services +- Humanitarian activity and aid +- Identity ecosystems +- Innovation incubation incentives + - Public investment + - Private capital +- Local regulations and laws +- National policies +- Public health services +- Records (birth, death, land etc) +``` +""" + +[config] +profiles = [ "basic", "document",] + +[config.custom_profiles.markmap] +args_config = "default" +plugins = [ "markmap",] +[expected_output] +basic = """

Markmap

+

+# Digital Identifiers and Rights
+
+## Community and Outreach
+
+- Experts
+- Grant organizations
+- IT ministries
+- Journalism schools
+- Journalists and reporters
+- Partnerships
+  - Patrons
+  - Sponsors
+- Policymakers
+- Rights activists
+- Startups
+- Thinktanks
+- Venture capital
+- Volunteers
+
+## Domains
+
+- Border controls
+- Citizenship
+- Digital data trusts
+- FinTech
+- Government
+- Health services delivery
+- Hospitality
+- Law enforcement
+- Online retail and commerce
+- Smart automation
+- Social media
+- Travel and tourism
+
+## Location
+
+- International
+- Local or domestic
+- Transit
+
+## Output and Outcomes
+
+- Best practices guide for product/service development
+- Conference
+- Conversations (eg. Twitter Spaces)
+- Masterclass webinars
+- Proceedings (talk playlist)
+- Reports
+- Review of Policies
+
+## Themes
+
+### Digital Identity
+
+- Anonymity
+- Architecture of digital trust
+- Control and ownership
+- Identity and identifier models
+- Inclusion and exclusion
+- Portability
+- Principles
+- Regulations
+- Reputation
+- Rights and agency
+- Trust framework
+- Verifiability
+- Vulnerable communities
+
+### Digital Rights
+
+- Current state across region
+- Harms
+- Emerging regulatory requirements
+- Web 3.0 and decentralization
+- Naturalization
+
+## Streams
+
+- Banking and finance
+- Data exchange and interoperability
+- Data governance models
+- Data markets
+- Digital identifiers and identity systems
+- Digital public goods
+- Digital public services
+- Humanitarian activity and aid
+- Identity ecosystems
+- Innovation incubation incentives
+  - Public investment
+  - Private capital
+- Local regulations and laws
+- National policies
+- Public health services
+- Records (birth, death, land etc)
+
+""" +document = """

Markmap

+
+# Digital Identifiers and Rights + +## Community and Outreach + +- Experts +- Grant organizations +- IT ministries +- Journalism schools +- Journalists and reporters +- Partnerships +- Patrons +- Sponsors +- Policymakers +- Rights activists +- Startups +- Thinktanks +- Venture capital +- Volunteers + +## Domains + +- Border controls +- Citizenship +- Digital data trusts +- FinTech +- Government +- Health services delivery +- Hospitality +- Law enforcement +- Online retail and commerce +- Smart automation +- Social media +- Travel and tourism + +## Location + +- International +- Local or domestic +- Transit + +## Output and Outcomes + +- Best practices guide for product/service development +- Conference +- Conversations (eg. Twitter Spaces) +- Masterclass webinars +- Proceedings (talk playlist) +- Reports +- Review of Policies + +## Themes + +### Digital Identity + +- Anonymity +- Architecture of digital trust +- Control and ownership +- Identity and identifier models +- Inclusion and exclusion +- Portability +- Principles +- Regulations +- Reputation +- Rights and agency +- Trust framework +- Verifiability +- Vulnerable communities + +### Digital Rights + +- Current state across region +- Harms +- Emerging regulatory requirements +- Web 3.0 and decentralization +- Naturalization + +## Streams + +- Banking and finance +- Data exchange and interoperability +- Data governance models +- Data markets +- Digital identifiers and identity systems +- Digital public goods +- Digital public services +- Humanitarian activity and aid +- Identity ecosystems +- Innovation incubation incentives +- Public investment +- Private capital +- Local regulations and laws +- National policies +- Public health services +- Records (birth, death, land etc) +
+""" +markmap = """

Markmap

+
+# Digital Identifiers and Rights + +## Community and Outreach + +- Experts +- Grant organizations +- IT ministries +- Journalism schools +- Journalists and reporters +- Partnerships +- Patrons +- Sponsors +- Policymakers +- Rights activists +- Startups +- Thinktanks +- Venture capital +- Volunteers + +## Domains + +- Border controls +- Citizenship +- Digital data trusts +- FinTech +- Government +- Health services delivery +- Hospitality +- Law enforcement +- Online retail and commerce +- Smart automation +- Social media +- Travel and tourism + +## Location + +- International +- Local or domestic +- Transit + +## Output and Outcomes + +- Best practices guide for product/service development +- Conference +- Conversations (eg. Twitter Spaces) +- Masterclass webinars +- Proceedings (talk playlist) +- Reports +- Review of Policies + +## Themes + +### Digital Identity + +- Anonymity +- Architecture of digital trust +- Control and ownership +- Identity and identifier models +- Inclusion and exclusion +- Portability +- Principles +- Regulations +- Reputation +- Rights and agency +- Trust framework +- Verifiability +- Vulnerable communities + +### Digital Rights + +- Current state across region +- Harms +- Emerging regulatory requirements +- Web 3.0 and decentralization +- Naturalization + +## Streams + +- Banking and finance +- Data exchange and interoperability +- Data governance models +- Data markets +- Digital identifiers and identity systems +- Digital public goods +- Digital public services +- Humanitarian activity and aid +- Identity ecosystems +- Innovation incubation incentives +- Public investment +- Private capital +- Local regulations and laws +- National policies +- Public health services +- Records (birth, death, land etc) +
+""" diff --git a/tests/data/markdown/mermaid.toml b/tests/data/markdown/mermaid.toml new file mode 100644 index 000000000..ab0d577ce --- /dev/null +++ b/tests/data/markdown/mermaid.toml @@ -0,0 +1,450 @@ +markdown = """ +# mermaid tests + +## Flowchart + +```{mermaid} +graph TD + A[Christmas] -->|Get money| B(Go shopping) + B --> C{Let me think} + C -->|One| D[Laptop] + C -->|Two| E[iPhone] + C -->|Three| F[fa:fa-car Car] +``` + +## Sequence Diagrams + +``` {mermaid} +sequenceDiagram + Alice->>+John: Hello John, how are you? + Alice->>+John: John, can you hear me? + John-->>-Alice: Hi Alice, I can hear you! + John-->>-Alice: I feel great! +``` + +## Class Diagram + +``` {mermaid} +classDiagram + Animal <|-- Duck + Animal <|-- Fish + Animal <|-- Zebra + Animal : +int age + Animal : +String gender + Animal: +isMammal() + Animal: +mate() + class Duck{ + +String beakColor + +swim() + +quack() + } + class Fish{ + -int sizeInFeet + -canEat() + } + class Zebra{ + +bool is_wild + +run() + } +``` + +## State Diagram + +``` {mermaid} +stateDiagram-v2 + [*] --> Still + Still --> [*] + Still --> Moving + Moving --> Still + Moving --> Crash + Crash --> [*] +``` + +## Gantt Chart + +``` {mermaid} + title A Gantt Diagram + dateFormat YYYY-MM-DD + section Section + A task :a1, 2014-01-01, 30d + Another task :after a1 , 20d + section Another + Task in sec :2014-01-12 , 12d + another task : 24d +``` + +## Pie Chart + +``` {mermaid} +pie title Pets adopted by volunteers + "Dogs" : 386 + "Cats" : 85 + "Rats" : 15 + +``` + +## ER Diagram + +```{mermaid} +erDiagram + CUSTOMER }|..|{ DELIVERY-ADDRESS : has + CUSTOMER ||--o{ ORDER : places + CUSTOMER ||--o{ INVOICE : "liable for" + DELIVERY-ADDRESS ||--o{ ORDER : receives + INVOICE ||--|{ ORDER : covers + ORDER ||--|{ ORDER-ITEM : includes + PRODUCT-CATEGORY ||--|{ PRODUCT : contains + PRODUCT ||--o{ ORDER-ITEM : "ordered in" +``` + +## User Journey + +``` {mermaid} + journey + title My working day + section Go to work + Make tea: 5: Me + Go upstairs: 3: Me + Do work: 1: Me, Cat + section Go home + Go downstairs: 5: Me + Sit down: 3: Me + +``` + +## Git Graph + +``` {mermaid} + gitGraph + commit + commit + branch develop + checkout develop + commit + commit + checkout main + merge develop + commit + commit + +``` +""" + +[config] +profiles = [ "basic", "document",] + +[config.custom_profiles.mermaid] +args_config = "default" +plugins = [ "mermaid",] + + +[expected_output] +basic = """

mermaid tests

+

Flowchart

+
graph TD
+    A[Christmas] -->|Get money| B(Go shopping)
+    B --> C{Let me think}
+    C -->|One| D[Laptop]
+    C -->|Two| E[iPhone]
+    C -->|Three| F[fa:fa-car Car]
+
+

Sequence Diagrams

+
sequenceDiagram
+    Alice->>+John: Hello John, how are you?
+    Alice->>+John: John, can you hear me?
+    John-->>-Alice: Hi Alice, I can hear you!
+    John-->>-Alice: I feel great!
+
+

Class Diagram

+
classDiagram
+    Animal <|-- Duck
+    Animal <|-- Fish
+    Animal <|-- Zebra
+    Animal : +int age
+    Animal : +String gender
+    Animal: +isMammal()
+    Animal: +mate()
+    class Duck{
+      +String beakColor
+      +swim()
+      +quack()
+    }
+    class Fish{
+      -int sizeInFeet
+      -canEat()
+    }
+    class Zebra{
+      +bool is_wild
+      +run()
+    }
+
+

State Diagram

+
stateDiagram-v2
+    [*] --> Still
+    Still --> [*]
+    Still --> Moving
+    Moving --> Still
+    Moving --> Crash
+    Crash --> [*]
+
+

Gantt Chart

+
    title A Gantt Diagram
+    dateFormat  YYYY-MM-DD
+    section Section
+    A task :a1, 2014-01-01, 30d
+    Another task :after a1  , 20d
+    section Another
+    Task in sec :2014-01-12  , 12d
+    another task : 24d
+
+

Pie Chart

+
pie title Pets adopted by volunteers
+    "Dogs" : 386
+    "Cats" : 85
+    "Rats" : 15
+
+
+

ER Diagram

+
erDiagram
+          CUSTOMER }|..|{ DELIVERY-ADDRESS : has
+          CUSTOMER ||--o{ ORDER : places
+          CUSTOMER ||--o{ INVOICE : "liable for"
+          DELIVERY-ADDRESS ||--o{ ORDER : receives
+          INVOICE ||--|{ ORDER : covers
+          ORDER ||--|{ ORDER-ITEM : includes
+          PRODUCT-CATEGORY ||--|{ PRODUCT : contains
+          PRODUCT ||--o{ ORDER-ITEM : "ordered in"
+
+

User Journey

+
  journey
+    title My working day
+    section Go to work
+      Make tea: 5: Me
+      Go upstairs: 3: Me
+      Do work: 1: Me, Cat
+    section Go home
+      Go downstairs: 5: Me
+      Sit down: 3: Me
+
+
+

Git Graph

+
    gitGraph
+      commit
+      commit
+      branch develop
+      checkout develop
+      commit
+      commit
+      checkout main
+      merge develop
+      commit
+      commit
+
+
+""" +document = """

mermaid tests

+

Flowchart

+
graph TD + A[Christmas] -->|Get money| B(Go shopping) + B --> C{Let me think} + C -->|One| D[Laptop] + C -->|Two| E[iPhone] + C -->|Three| F[fa:fa-car Car] +
+

Sequence Diagrams

+
sequenceDiagram + Alice->>+John: Hello John, how are you? + Alice->>+John: John, can you hear me? + John-->>-Alice: Hi Alice, I can hear you! + John-->>-Alice: I feel great! +
+

Class Diagram

+
classDiagram + Animal <|-- Duck + Animal <|-- Fish + Animal <|-- Zebra + Animal : +int age + Animal : +String gender + Animal: +isMammal() + Animal: +mate() + class Duck{ + +String beakColor + +swim() + +quack() + } + class Fish{ + -int sizeInFeet + -canEat() + } + class Zebra{ + +bool is_wild + +run() + } +
+

State Diagram

+
stateDiagram-v2 + [*] --> Still + Still --> [*] + Still --> Moving + Moving --> Still + Moving --> Crash + Crash --> [*] +
+

Gantt Chart

+
title A Gantt Diagram + dateFormat YYYY-MM-DD + section Section + A task :a1, 2014-01-01, 30d + Another task :after a1 , 20d + section Another + Task in sec :2014-01-12 , 12d + another task : 24d +
+

Pie Chart

+
pie title Pets adopted by volunteers + "Dogs" : 386 + "Cats" : 85 + "Rats" : 15 + +
+

ER Diagram

+
erDiagram + CUSTOMER }|..|{ DELIVERY-ADDRESS : has + CUSTOMER ||--o{ ORDER : places + CUSTOMER ||--o{ INVOICE : "liable for" + DELIVERY-ADDRESS ||--o{ ORDER : receives + INVOICE ||--|{ ORDER : covers + ORDER ||--|{ ORDER-ITEM : includes + PRODUCT-CATEGORY ||--|{ PRODUCT : contains + PRODUCT ||--o{ ORDER-ITEM : "ordered in" +
+

User Journey

+
journey + title My working day + section Go to work + Make tea: 5: Me + Go upstairs: 3: Me + Do work: 1: Me, Cat + section Go home + Go downstairs: 5: Me + Sit down: 3: Me + +
+

Git Graph

+
gitGraph + commit + commit + branch develop + checkout develop + commit + commit + checkout main + merge develop + commit + commit + +
+""" +mermaid = """

mermaid tests

+

Flowchart

+
graph TD + A[Christmas] -->|Get money| B(Go shopping) + B --> C{Let me think} + C -->|One| D[Laptop] + C -->|Two| E[iPhone] + C -->|Three| F[fa:fa-car Car] +
+

Sequence Diagrams

+
sequenceDiagram + Alice->>+John: Hello John, how are you? + Alice->>+John: John, can you hear me? + John-->>-Alice: Hi Alice, I can hear you! + John-->>-Alice: I feel great! +
+

Class Diagram

+
classDiagram + Animal <|-- Duck + Animal <|-- Fish + Animal <|-- Zebra + Animal : +int age + Animal : +String gender + Animal: +isMammal() + Animal: +mate() + class Duck{ + +String beakColor + +swim() + +quack() + } + class Fish{ + -int sizeInFeet + -canEat() + } + class Zebra{ + +bool is_wild + +run() + } +
+

State Diagram

+
stateDiagram-v2 + [*] --> Still + Still --> [*] + Still --> Moving + Moving --> Still + Moving --> Crash + Crash --> [*] +
+

Gantt Chart

+
title A Gantt Diagram + dateFormat YYYY-MM-DD + section Section + A task :a1, 2014-01-01, 30d + Another task :after a1 , 20d + section Another + Task in sec :2014-01-12 , 12d + another task : 24d +
+

Pie Chart

+
pie title Pets adopted by volunteers + "Dogs" : 386 + "Cats" : 85 + "Rats" : 15 + +
+

ER Diagram

+
erDiagram + CUSTOMER }|..|{ DELIVERY-ADDRESS : has + CUSTOMER ||--o{ ORDER : places + CUSTOMER ||--o{ INVOICE : "liable for" + DELIVERY-ADDRESS ||--o{ ORDER : receives + INVOICE ||--|{ ORDER : covers + ORDER ||--|{ ORDER-ITEM : includes + PRODUCT-CATEGORY ||--|{ PRODUCT : contains + PRODUCT ||--o{ ORDER-ITEM : "ordered in" +
+

User Journey

+
journey + title My working day + section Go to work + Make tea: 5: Me + Go upstairs: 3: Me + Do work: 1: Me, Cat + section Go home + Go downstairs: 5: Me + Sit down: 3: Me + +
+

Git Graph

+
gitGraph + commit + commit + branch develop + checkout develop + commit + commit + checkout main + merge develop + commit + commit + +
+""" diff --git a/tests/data/markdown/vega-lite.toml b/tests/data/markdown/vega-lite.toml new file mode 100644 index 000000000..87ab709fd --- /dev/null +++ b/tests/data/markdown/vega-lite.toml @@ -0,0 +1,1369 @@ +markdown = """ +# vega-lite tests + +## Interactive Scatter Plot Matrix + +``` { vega-lite } +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "repeat": { + "row": ["Horsepower", "Acceleration", "Miles_per_Gallon"], + "column": ["Miles_per_Gallon", "Acceleration", "Horsepower"] + }, + "spec": { + "data": {"url": "https://vega.github.io/vega-lite/examples/data/cars.json"}, + "mark": "point", + "params": [ + { + "name": "brush", + "select": { + "type": "interval", + "resolve": "union", + "on": "[mousedown[event.shiftKey], window:mouseup] > window:mousemove!", + "translate": "[mousedown[event.shiftKey], window:mouseup] > window:mousemove!", + "zoom": "wheel![event.shiftKey]" + } + }, + { + "name": "grid", + "select": { + "type": "interval", + "resolve": "global", + "translate": "[mousedown[!event.shiftKey], window:mouseup] > window:mousemove!", + "zoom": "wheel![!event.shiftKey]" + }, + "bind": "scales" + } + ], + "encoding": { + "x": {"field": {"repeat": "column"}, "type": "quantitative"}, + "y": { + "field": {"repeat": "row"}, + "type": "quantitative", + "axis": {"minExtent": 30} + }, + "color": { + "condition": { + "param": "brush", + "field": "Origin", + "type": "nominal" + }, + "value": "grey" + } + } + } +} +``` + +## Population Pyramid + +```{vega-lite} +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "description": "A population pyramid for the US in 2000.", + "data": { "url": "https://vega.github.io/vega-lite/examples/data/population.json"}, + "transform": [ + {"filter": "datum.year == 2000"}, + {"calculate": "datum.sex == 2 ? 'Female' : 'Male'", "as": "gender"} + ], + "spacing": 0, + "hconcat": [{ + "transform": [{ + "filter": {"field": "gender", "equal": "Female"} + }], + "title": "Female", + "mark": "bar", + "encoding": { + "y": { + "field": "age", "axis": null, "sort": "descending" + }, + "x": { + "aggregate": "sum", "field": "people", + "title": "population", + "axis": {"format": "s"}, + "sort": "descending" + }, + "color": { + "field": "gender", + "scale": {"range": ["#675193", "#ca8861"]}, + "legend": null + } + } + }, { + "width": 20, + "view": {"stroke": null}, + "mark": { + "type": "text", + "align": "center" + }, + "encoding": { + "y": {"field": "age", "type": "ordinal", "axis": null, "sort": "descending"}, + "text": {"field": "age", "type": "quantitative"} + } + }, { + "transform": [{ + "filter": {"field": "gender", "equal": "Male"} + }], + "title": "Male", + "mark": "bar", + "encoding": { + "y": { + "field": "age", "title": null, + "axis": null, "sort": "descending" + }, + "x": { + "aggregate": "sum", "field": "people", + "title": "population", + "axis": {"format": "s"} + }, + "color": { + "field": "gender", + "legend": null + } + } + }], + "config": { + "view": {"stroke": null}, + "axis": {"grid": false} + } +} + +``` + +## Discretizing scales + +``` {vega-lite} +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "description": "Horizontally concatenated charts that show different types of discretizing scales.", + "data": { + "values": [ + {"a": "A", "b": 28}, + {"a": "B", "b": 55}, + {"a": "C", "b": 43}, + {"a": "D", "b": 91}, + {"a": "E", "b": 81}, + {"a": "F", "b": 53}, + {"a": "G", "b": 19}, + {"a": "H", "b": 87}, + {"a": "I", "b": 52} + ] + }, + "hconcat": [ + { + "mark": "circle", + "encoding": { + "y": { + "field": "b", + "type": "nominal", + "sort": null, + "axis": { + "ticks": false, + "domain": false, + "title": null + } + }, + "size": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantize" + } + }, + "color": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantize", + "zero": true + }, + "legend": { + "title": "Quantize" + } + } + } + }, + { + "mark": "circle", + "encoding": { + "y": { + "field": "b", + "type": "nominal", + "sort": null, + "axis": { + "ticks": false, + "domain": false, + "title": null + } + }, + "size": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantile", + "range": [80, 160, 240, 320, 400] + } + }, + "color": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantile", + "scheme": "magma" + }, + "legend": { + "format": "d", + "title": "Quantile" + } + } + } + }, + { + "mark": "circle", + "encoding": { + "y": { + "field": "b", + "type": "nominal", + "sort": null, + "axis": { + "ticks": false, + "domain": false, + "title": null + } + }, + "size": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "threshold", + "domain": [30, 70], + "range": [80, 200, 320] + } + }, + "color": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "threshold", + "domain": [30, 70], + "scheme": "viridis" + }, + "legend": { + "title": "Threshold" + } + } + } + } + ], + "resolve": { + "scale": { + "color": "independent", + "size": "independent" + } + } +} +``` + +## Marginal Histograms + +``` {vega-lite} +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "data": {"url": "https://vega.github.io/vega-lite/examples/data/movies.json"}, + "spacing": 15, + "bounds": "flush", + "vconcat": [{ + "mark": "bar", + "height": 60, + "encoding": { + "x": { + "bin": true, + "field": "IMDB Rating", + "axis": null + }, + "y": { + "aggregate": "count", + "scale": { + "domain": [0,1000] + }, + "title": "" + } + } + }, { + "spacing": 15, + "bounds": "flush", + "hconcat": [{ + "mark": "rect", + "encoding": { + "x": {"bin": true, "field": "IMDB Rating"}, + "y": {"bin": true, "field": "Rotten Tomatoes Rating"}, + "color": {"aggregate": "count"} + } + }, { + "mark": "bar", + "width": 60, + "encoding": { + "y": { + "bin": true, + "field": "Rotten Tomatoes Rating", + "axis": null + }, + "x": { + "aggregate": "count", + "scale": {"domain": [0,1000]}, + "title": "" + } + } + }] + }], + "config": { + "view": { + "stroke": "transparent" + } + } +} +``` + +## Radial Plot + +``` {vega-lite} +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "description": "A simple radial chart with embedded data.", + "data": { + "values": [12, 23, 47, 6, 52, 19] + }, + "layer": [{ + "mark": {"type": "arc", "innerRadius": 20, "stroke": "#fff"} + },{ + "mark": {"type": "text", "radiusOffset": 10}, + "encoding": { + "text": {"field": "data", "type": "quantitative"} + } + }], + "encoding": { + "theta": {"field": "data", "type": "quantitative", "stack": true}, + "radius": {"field": "data", "scale": {"type": "sqrt", "zero": true, "rangeMin": 20}}, + "color": {"field": "data", "type": "nominal", "legend": null} + } +} + +``` +""" + +[config] +profiles = [ "basic", "document",] + +[config.custom_profiles.markmap] +args_config = "default" +plugins = [ "vega-lite",] + +[expected_output] +basic = """

vega-lite tests

+

Interactive Scatter Plot Matrix

+
{
+  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+  "repeat": {
+    "row": ["Horsepower", "Acceleration", "Miles_per_Gallon"],
+    "column": ["Miles_per_Gallon", "Acceleration", "Horsepower"]
+  },
+  "spec": {
+    "data": {"url": "https://vega.github.io/vega-lite/examples/data/cars.json"},
+    "mark": "point",
+    "params": [
+      {
+        "name": "brush",
+        "select": {
+          "type": "interval",
+          "resolve": "union",
+          "on": "[mousedown[event.shiftKey], window:mouseup] > window:mousemove!",
+          "translate": "[mousedown[event.shiftKey], window:mouseup] > window:mousemove!",
+          "zoom": "wheel![event.shiftKey]"
+        }
+      },
+      {
+        "name": "grid",
+        "select": {
+          "type": "interval",
+          "resolve": "global",
+          "translate": "[mousedown[!event.shiftKey], window:mouseup] > window:mousemove!",
+          "zoom": "wheel![!event.shiftKey]"
+        },
+        "bind": "scales"
+      }
+    ],
+    "encoding": {
+      "x": {"field": {"repeat": "column"}, "type": "quantitative"},
+      "y": {
+        "field": {"repeat": "row"},
+        "type": "quantitative",
+        "axis": {"minExtent": 30}
+      },
+      "color": {
+        "condition": {
+          "param": "brush",
+          "field": "Origin",
+          "type": "nominal"
+        },
+        "value": "grey"
+      }
+    }
+  }
+}
+
+

Population Pyramid

+
{
+  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+  "description": "A population pyramid for the US in 2000.",
+  "data": { "url": "https://vega.github.io/vega-lite/examples/data/population.json"},
+  "transform": [
+    {"filter": "datum.year == 2000"},
+    {"calculate": "datum.sex == 2 ? 'Female' : 'Male'", "as": "gender"}
+  ],
+  "spacing": 0,
+  "hconcat": [{
+    "transform": [{
+      "filter": {"field": "gender", "equal": "Female"}
+    }],
+    "title": "Female",
+    "mark": "bar",
+    "encoding": {
+      "y": {
+        "field": "age", "axis": null, "sort": "descending"
+      },
+      "x": {
+        "aggregate": "sum", "field": "people",
+        "title": "population",
+        "axis": {"format": "s"},
+        "sort": "descending"
+      },
+      "color": {
+        "field": "gender",
+        "scale": {"range": ["#675193", "#ca8861"]},
+        "legend": null
+      }
+    }
+  }, {
+    "width": 20,
+    "view": {"stroke": null},
+    "mark": {
+      "type": "text",
+      "align": "center"
+    },
+    "encoding": {
+      "y": {"field": "age", "type": "ordinal", "axis": null, "sort": "descending"},
+      "text": {"field": "age", "type": "quantitative"}
+    }
+  }, {
+    "transform": [{
+      "filter": {"field": "gender", "equal": "Male"}
+    }],
+    "title": "Male",
+    "mark": "bar",
+    "encoding": {
+      "y": {
+        "field": "age", "title": null,
+        "axis": null, "sort": "descending"
+      },
+      "x": {
+        "aggregate": "sum", "field": "people",
+        "title": "population",
+        "axis": {"format": "s"}
+      },
+      "color": {
+        "field": "gender",
+        "legend": null
+      }
+    }
+  }],
+  "config": {
+    "view": {"stroke": null},
+    "axis": {"grid": false}
+  }
+}
+
+
+

Discretizing scales

+
{
+  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+  "description": "Horizontally concatenated charts that show different types of discretizing scales.",
+  "data": {
+    "values": [
+      {"a": "A", "b": 28},
+      {"a": "B", "b": 55},
+      {"a": "C", "b": 43},
+      {"a": "D", "b": 91},
+      {"a": "E", "b": 81},
+      {"a": "F", "b": 53},
+      {"a": "G", "b": 19},
+      {"a": "H", "b": 87},
+      {"a": "I", "b": 52}
+    ]
+  },
+  "hconcat": [
+    {
+      "mark": "circle",
+      "encoding": {
+        "y": {
+          "field": "b",
+          "type": "nominal",
+          "sort": null,
+          "axis": {
+            "ticks": false,
+            "domain": false,
+            "title": null
+          }
+        },
+        "size": {
+          "field": "b",
+          "type": "quantitative",
+          "scale": {
+            "type": "quantize"
+          }
+        },
+        "color": {
+          "field": "b",
+          "type": "quantitative",
+          "scale": {
+            "type": "quantize",
+            "zero": true
+          },
+          "legend": {
+            "title": "Quantize"
+          }
+        }
+      }
+    },
+    {
+      "mark": "circle",
+      "encoding": {
+        "y": {
+          "field": "b",
+          "type": "nominal",
+          "sort": null,
+          "axis": {
+            "ticks": false,
+            "domain": false,
+            "title": null
+          }
+        },
+        "size": {
+          "field": "b",
+          "type": "quantitative",
+          "scale": {
+            "type": "quantile",
+            "range": [80, 160, 240, 320, 400]
+          }
+        },
+        "color": {
+          "field": "b",
+          "type": "quantitative",
+          "scale": {
+            "type": "quantile",
+            "scheme": "magma"
+          },
+          "legend": {
+            "format": "d",
+            "title": "Quantile"
+          }
+        }
+      }
+    },
+    {
+      "mark": "circle",
+      "encoding": {
+        "y": {
+          "field": "b",
+          "type": "nominal",
+          "sort": null,
+          "axis": {
+            "ticks": false,
+            "domain": false,
+            "title": null
+          }
+        },
+        "size": {
+          "field": "b",
+          "type": "quantitative",
+          "scale": {
+            "type": "threshold",
+            "domain": [30, 70],
+            "range": [80, 200, 320]
+          }
+        },
+        "color": {
+          "field": "b",
+          "type": "quantitative",
+          "scale": {
+            "type": "threshold",
+            "domain": [30, 70],
+            "scheme": "viridis"
+          },
+          "legend": {
+            "title": "Threshold"
+          }
+        }
+      }
+    }
+  ],
+  "resolve": {
+    "scale": {
+      "color": "independent",
+      "size": "independent"
+    }
+  }
+}
+
+

Marginal Histograms

+
{
+  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+  "data": {"url": "https://vega.github.io/vega-lite/examples/data/movies.json"},
+  "spacing": 15,
+  "bounds": "flush",
+  "vconcat": [{
+    "mark": "bar",
+    "height": 60,
+    "encoding": {
+      "x": {
+        "bin": true,
+        "field": "IMDB Rating",
+        "axis": null
+      },
+      "y": {
+        "aggregate": "count",
+        "scale": {
+          "domain": [0,1000]
+        },
+        "title": ""
+      }
+    }
+  }, {
+    "spacing": 15,
+    "bounds": "flush",
+    "hconcat": [{
+      "mark": "rect",
+      "encoding": {
+        "x": {"bin": true, "field": "IMDB Rating"},
+        "y": {"bin": true, "field": "Rotten Tomatoes Rating"},
+        "color": {"aggregate": "count"}
+      }
+    }, {
+      "mark": "bar",
+      "width": 60,
+      "encoding": {
+        "y": {
+          "bin": true,
+          "field": "Rotten Tomatoes Rating",
+          "axis": null
+        },
+        "x": {
+          "aggregate": "count",
+          "scale": {"domain": [0,1000]},
+          "title": ""
+        }
+      }
+    }]
+  }],
+  "config": {
+    "view": {
+      "stroke": "transparent"
+    }
+  }
+}
+
+

Radial Plot

+
{
+  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+  "description": "A simple radial chart with embedded data.",
+  "data": {
+    "values": [12, 23, 47, 6, 52, 19]
+  },
+  "layer": [{
+    "mark": {"type": "arc", "innerRadius": 20, "stroke": "#fff"}
+  },{
+    "mark": {"type": "text", "radiusOffset": 10},
+    "encoding": {
+      "text": {"field": "data", "type": "quantitative"}
+    }
+  }],
+  "encoding": {
+    "theta": {"field": "data", "type": "quantitative", "stack": true},
+    "radius": {"field": "data", "scale": {"type": "sqrt", "zero": true, "rangeMin": 20}},
+    "color": {"field": "data", "type": "nominal", "legend": null}
+  }
+}
+
+
+""" +document = """

vega-lite tests

+

Interactive Scatter Plot Matrix

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"repeat": { + "row": ["Horsepower", "Acceleration", "Miles_per_Gallon"], + "column": ["Miles_per_Gallon", "Acceleration", "Horsepower"] +}, +"spec": { + "data": {"url": "https://vega.github.io/vega-lite/examples/data/cars.json"}, + "mark": "point", + "params": [ + { + "name": "brush", + "select": { + "type": "interval", + "resolve": "union", + "on": "[mousedown[event.shiftKey], window:mouseup] > window:mousemove!", + "translate": "[mousedown[event.shiftKey], window:mouseup] > window:mousemove!", + "zoom": "wheel![event.shiftKey]" + } + }, + { + "name": "grid", + "select": { + "type": "interval", + "resolve": "global", + "translate": "[mousedown[!event.shiftKey], window:mouseup] > window:mousemove!", + "zoom": "wheel![!event.shiftKey]" + }, + "bind": "scales" + } + ], + "encoding": { + "x": {"field": {"repeat": "column"}, "type": "quantitative"}, + "y": { + "field": {"repeat": "row"}, + "type": "quantitative", + "axis": {"minExtent": 30} + }, + "color": { + "condition": { + "param": "brush", + "field": "Origin", + "type": "nominal" + }, + "value": "grey" + } + } +} +} +
+

Population Pyramid

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"description": "A population pyramid for the US in 2000.", +"data": { "url": "https://vega.github.io/vega-lite/examples/data/population.json"}, +"transform": [ + {"filter": "datum.year == 2000"}, + {"calculate": "datum.sex == 2 ? 'Female' : 'Male'", "as": "gender"} +], +"spacing": 0, +"hconcat": [{ + "transform": [{ + "filter": {"field": "gender", "equal": "Female"} + }], + "title": "Female", + "mark": "bar", + "encoding": { + "y": { + "field": "age", "axis": null, "sort": "descending" + }, + "x": { + "aggregate": "sum", "field": "people", + "title": "population", + "axis": {"format": "s"}, + "sort": "descending" + }, + "color": { + "field": "gender", + "scale": {"range": ["#675193", "#ca8861"]}, + "legend": null + } + } +}, { + "width": 20, + "view": {"stroke": null}, + "mark": { + "type": "text", + "align": "center" + }, + "encoding": { + "y": {"field": "age", "type": "ordinal", "axis": null, "sort": "descending"}, + "text": {"field": "age", "type": "quantitative"} + } +}, { + "transform": [{ + "filter": {"field": "gender", "equal": "Male"} + }], + "title": "Male", + "mark": "bar", + "encoding": { + "y": { + "field": "age", "title": null, + "axis": null, "sort": "descending" + }, + "x": { + "aggregate": "sum", "field": "people", + "title": "population", + "axis": {"format": "s"} + }, + "color": { + "field": "gender", + "legend": null + } + } +}], +"config": { + "view": {"stroke": null}, + "axis": {"grid": false} +} +} + +
+

Discretizing scales

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"description": "Horizontally concatenated charts that show different types of discretizing scales.", +"data": { + "values": [ + {"a": "A", "b": 28}, + {"a": "B", "b": 55}, + {"a": "C", "b": 43}, + {"a": "D", "b": 91}, + {"a": "E", "b": 81}, + {"a": "F", "b": 53}, + {"a": "G", "b": 19}, + {"a": "H", "b": 87}, + {"a": "I", "b": 52} + ] +}, +"hconcat": [ + { + "mark": "circle", + "encoding": { + "y": { + "field": "b", + "type": "nominal", + "sort": null, + "axis": { + "ticks": false, + "domain": false, + "title": null + } + }, + "size": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantize" + } + }, + "color": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantize", + "zero": true + }, + "legend": { + "title": "Quantize" + } + } + } + }, + { + "mark": "circle", + "encoding": { + "y": { + "field": "b", + "type": "nominal", + "sort": null, + "axis": { + "ticks": false, + "domain": false, + "title": null + } + }, + "size": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantile", + "range": [80, 160, 240, 320, 400] + } + }, + "color": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantile", + "scheme": "magma" + }, + "legend": { + "format": "d", + "title": "Quantile" + } + } + } + }, + { + "mark": "circle", + "encoding": { + "y": { + "field": "b", + "type": "nominal", + "sort": null, + "axis": { + "ticks": false, + "domain": false, + "title": null + } + }, + "size": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "threshold", + "domain": [30, 70], + "range": [80, 200, 320] + } + }, + "color": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "threshold", + "domain": [30, 70], + "scheme": "viridis" + }, + "legend": { + "title": "Threshold" + } + } + } + } +], +"resolve": { + "scale": { + "color": "independent", + "size": "independent" + } +} +} +
+

Marginal Histograms

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"data": {"url": "https://vega.github.io/vega-lite/examples/data/movies.json"}, +"spacing": 15, +"bounds": "flush", +"vconcat": [{ + "mark": "bar", + "height": 60, + "encoding": { + "x": { + "bin": true, + "field": "IMDB Rating", + "axis": null + }, + "y": { + "aggregate": "count", + "scale": { + "domain": [0,1000] + }, + "title": "" + } + } +}, { + "spacing": 15, + "bounds": "flush", + "hconcat": [{ + "mark": "rect", + "encoding": { + "x": {"bin": true, "field": "IMDB Rating"}, + "y": {"bin": true, "field": "Rotten Tomatoes Rating"}, + "color": {"aggregate": "count"} + } + }, { + "mark": "bar", + "width": 60, + "encoding": { + "y": { + "bin": true, + "field": "Rotten Tomatoes Rating", + "axis": null + }, + "x": { + "aggregate": "count", + "scale": {"domain": [0,1000]}, + "title": "" + } + } + }] +}], +"config": { + "view": { + "stroke": "transparent" + } +} +} +
+

Radial Plot

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"description": "A simple radial chart with embedded data.", +"data": { + "values": [12, 23, 47, 6, 52, 19] +}, +"layer": [{ + "mark": {"type": "arc", "innerRadius": 20, "stroke": "#fff"} +},{ + "mark": {"type": "text", "radiusOffset": 10}, + "encoding": { + "text": {"field": "data", "type": "quantitative"} + } +}], +"encoding": { + "theta": {"field": "data", "type": "quantitative", "stack": true}, + "radius": {"field": "data", "scale": {"type": "sqrt", "zero": true, "rangeMin": 20}}, + "color": {"field": "data", "type": "nominal", "legend": null} +} +} + +
+""" +markmap = """

vega-lite tests

+

Interactive Scatter Plot Matrix

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"repeat": { + "row": ["Horsepower", "Acceleration", "Miles_per_Gallon"], + "column": ["Miles_per_Gallon", "Acceleration", "Horsepower"] +}, +"spec": { + "data": {"url": "https://vega.github.io/vega-lite/examples/data/cars.json"}, + "mark": "point", + "params": [ + { + "name": "brush", + "select": { + "type": "interval", + "resolve": "union", + "on": "[mousedown[event.shiftKey], window:mouseup] > window:mousemove!", + "translate": "[mousedown[event.shiftKey], window:mouseup] > window:mousemove!", + "zoom": "wheel![event.shiftKey]" + } + }, + { + "name": "grid", + "select": { + "type": "interval", + "resolve": "global", + "translate": "[mousedown[!event.shiftKey], window:mouseup] > window:mousemove!", + "zoom": "wheel![!event.shiftKey]" + }, + "bind": "scales" + } + ], + "encoding": { + "x": {"field": {"repeat": "column"}, "type": "quantitative"}, + "y": { + "field": {"repeat": "row"}, + "type": "quantitative", + "axis": {"minExtent": 30} + }, + "color": { + "condition": { + "param": "brush", + "field": "Origin", + "type": "nominal" + }, + "value": "grey" + } + } +} +} +
+

Population Pyramid

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"description": "A population pyramid for the US in 2000.", +"data": { "url": "https://vega.github.io/vega-lite/examples/data/population.json"}, +"transform": [ + {"filter": "datum.year == 2000"}, + {"calculate": "datum.sex == 2 ? 'Female' : 'Male'", "as": "gender"} +], +"spacing": 0, +"hconcat": [{ + "transform": [{ + "filter": {"field": "gender", "equal": "Female"} + }], + "title": "Female", + "mark": "bar", + "encoding": { + "y": { + "field": "age", "axis": null, "sort": "descending" + }, + "x": { + "aggregate": "sum", "field": "people", + "title": "population", + "axis": {"format": "s"}, + "sort": "descending" + }, + "color": { + "field": "gender", + "scale": {"range": ["#675193", "#ca8861"]}, + "legend": null + } + } +}, { + "width": 20, + "view": {"stroke": null}, + "mark": { + "type": "text", + "align": "center" + }, + "encoding": { + "y": {"field": "age", "type": "ordinal", "axis": null, "sort": "descending"}, + "text": {"field": "age", "type": "quantitative"} + } +}, { + "transform": [{ + "filter": {"field": "gender", "equal": "Male"} + }], + "title": "Male", + "mark": "bar", + "encoding": { + "y": { + "field": "age", "title": null, + "axis": null, "sort": "descending" + }, + "x": { + "aggregate": "sum", "field": "people", + "title": "population", + "axis": {"format": "s"} + }, + "color": { + "field": "gender", + "legend": null + } + } +}], +"config": { + "view": {"stroke": null}, + "axis": {"grid": false} +} +} + +
+

Discretizing scales

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"description": "Horizontally concatenated charts that show different types of discretizing scales.", +"data": { + "values": [ + {"a": "A", "b": 28}, + {"a": "B", "b": 55}, + {"a": "C", "b": 43}, + {"a": "D", "b": 91}, + {"a": "E", "b": 81}, + {"a": "F", "b": 53}, + {"a": "G", "b": 19}, + {"a": "H", "b": 87}, + {"a": "I", "b": 52} + ] +}, +"hconcat": [ + { + "mark": "circle", + "encoding": { + "y": { + "field": "b", + "type": "nominal", + "sort": null, + "axis": { + "ticks": false, + "domain": false, + "title": null + } + }, + "size": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantize" + } + }, + "color": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantize", + "zero": true + }, + "legend": { + "title": "Quantize" + } + } + } + }, + { + "mark": "circle", + "encoding": { + "y": { + "field": "b", + "type": "nominal", + "sort": null, + "axis": { + "ticks": false, + "domain": false, + "title": null + } + }, + "size": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantile", + "range": [80, 160, 240, 320, 400] + } + }, + "color": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "quantile", + "scheme": "magma" + }, + "legend": { + "format": "d", + "title": "Quantile" + } + } + } + }, + { + "mark": "circle", + "encoding": { + "y": { + "field": "b", + "type": "nominal", + "sort": null, + "axis": { + "ticks": false, + "domain": false, + "title": null + } + }, + "size": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "threshold", + "domain": [30, 70], + "range": [80, 200, 320] + } + }, + "color": { + "field": "b", + "type": "quantitative", + "scale": { + "type": "threshold", + "domain": [30, 70], + "scheme": "viridis" + }, + "legend": { + "title": "Threshold" + } + } + } + } +], +"resolve": { + "scale": { + "color": "independent", + "size": "independent" + } +} +} +
+

Marginal Histograms

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"data": {"url": "https://vega.github.io/vega-lite/examples/data/movies.json"}, +"spacing": 15, +"bounds": "flush", +"vconcat": [{ + "mark": "bar", + "height": 60, + "encoding": { + "x": { + "bin": true, + "field": "IMDB Rating", + "axis": null + }, + "y": { + "aggregate": "count", + "scale": { + "domain": [0,1000] + }, + "title": "" + } + } +}, { + "spacing": 15, + "bounds": "flush", + "hconcat": [{ + "mark": "rect", + "encoding": { + "x": {"bin": true, "field": "IMDB Rating"}, + "y": {"bin": true, "field": "Rotten Tomatoes Rating"}, + "color": {"aggregate": "count"} + } + }, { + "mark": "bar", + "width": 60, + "encoding": { + "y": { + "bin": true, + "field": "Rotten Tomatoes Rating", + "axis": null + }, + "x": { + "aggregate": "count", + "scale": {"domain": [0,1000]}, + "title": "" + } + } + }] +}], +"config": { + "view": { + "stroke": "transparent" + } +} +} +
+

Radial Plot

+
{ +"$schema": "https://vega.github.io/schema/vega-lite/v5.json", +"description": "A simple radial chart with embedded data.", +"data": { + "values": [12, 23, 47, 6, 52, 19] +}, +"layer": [{ + "mark": {"type": "arc", "innerRadius": 20, "stroke": "#fff"} +},{ + "mark": {"type": "text", "radiusOffset": 10}, + "encoding": { + "text": {"field": "data", "type": "quantitative"} + } +}], +"encoding": { + "theta": {"field": "data", "type": "quantitative", "stack": true}, + "radius": {"field": "data", "scale": {"type": "sqrt", "zero": true, "rangeMin": 20}}, + "color": {"field": "data", "type": "nominal", "legend": null} +} +} + +
+""" From c433779025ea59522e03da3566f482bc1202a635 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 8 Nov 2022 18:56:18 +0530 Subject: [PATCH 30/72] Tests for embeddable fenced blocks for markmap, vega-lite and mermaid #1496 --- tests/data/markdown/vega-lite.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/markdown/vega-lite.toml b/tests/data/markdown/vega-lite.toml index 87ab709fd..c2085e2ae 100644 --- a/tests/data/markdown/vega-lite.toml +++ b/tests/data/markdown/vega-lite.toml @@ -354,7 +354,7 @@ markdown = """ [config] profiles = [ "basic", "document",] -[config.custom_profiles.markmap] +[config.custom_profiles.vega-lite] args_config = "default" plugins = [ "vega-lite",] @@ -1031,7 +1031,7 @@ document = """

vega-lite tests vega-lite tests

+vega-lite = """

vega-lite tests

Interactive Scatter Plot Matrix

{ "$schema": "https://vega.github.io/schema/vega-lite/v5.json", From d41ece8b1e468578aff0b3595d994cf284160068 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 14 Nov 2022 16:47:33 +0530 Subject: [PATCH 31/72] Introduce prismjs to render fenced blocks with language-* output. #1515 --- funnel/assets/js/index.js | 2 ++ funnel/assets/js/project.js | 2 ++ funnel/assets/js/schedule_view.js | 2 ++ funnel/assets/js/submission.js | 2 ++ funnel/assets/js/utils/prism.js | 52 +++++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+) create mode 100644 funnel/assets/js/utils/prism.js diff --git a/funnel/assets/js/index.js b/funnel/assets/js/index.js index d5db5d492..ed18eaf2d 100644 --- a/funnel/assets/js/index.js +++ b/funnel/assets/js/index.js @@ -2,6 +2,7 @@ import SaveProject from './utils/bookmark'; import 'htmx.org'; import MarkmapEmbed from './utils/markmap'; import MermaidEmbed from './utils/mermaid'; +import PrismEmbed from './utils/prism'; $(() => { window.Hasgeek.homeInit = function homeInit() { @@ -23,4 +24,5 @@ $(() => { }; MarkmapEmbed.init(); MermaidEmbed.init(); + PrismEmbed.init(); }); diff --git a/funnel/assets/js/project.js b/funnel/assets/js/project.js index 315e0fa4f..9ce007550 100644 --- a/funnel/assets/js/project.js +++ b/funnel/assets/js/project.js @@ -2,6 +2,7 @@ import L from 'leaflet'; import TypeformEmbed from './utils/typeform_embed'; import MarkmapEmbed from './utils/markmap'; import MermaidEmbed from './utils/mermaid'; +import PrismEmbed from './utils/prism'; const EmbedMap = { init({ mapId, latitude, longitude }) { @@ -55,5 +56,6 @@ $(() => { TypeformEmbed.init('#about .markdown'); MarkmapEmbed.init(); MermaidEmbed.init(); + PrismEmbed.init(); }; }); diff --git a/funnel/assets/js/schedule_view.js b/funnel/assets/js/schedule_view.js index 30a335423..0c7248402 100644 --- a/funnel/assets/js/schedule_view.js +++ b/funnel/assets/js/schedule_view.js @@ -7,6 +7,7 @@ import Spa from './utils/spahelper'; import Utils from './utils/helper'; import MarkmapEmbed from './utils/markmap'; import MermaidEmbed from './utils/mermaid'; +import PrismEmbed from './utils/prism'; const Schedule = { renderScheduleTable() { @@ -150,6 +151,7 @@ const Schedule = { Utils.enableWebShare(); MarkmapEmbed.init(); MermaidEmbed.init(); + PrismEmbed.init(); observer.disconnect(); } }); diff --git a/funnel/assets/js/submission.js b/funnel/assets/js/submission.js index 0839a5780..c0ec82002 100644 --- a/funnel/assets/js/submission.js +++ b/funnel/assets/js/submission.js @@ -4,6 +4,7 @@ import addVegaSupport from './utils/vegaembed'; import TypeformEmbed from './utils/typeform_embed'; import MarkmapEmbed from './utils/markmap'; import MermaidEmbed from './utils/mermaid'; +import PrismEmbed from './utils/prism'; export const Submission = { init() { @@ -129,4 +130,5 @@ $(() => { TypeformEmbed.init('#submission .markdown'); MarkmapEmbed.init(); MermaidEmbed.init(); + PrismEmbed.init(); }); diff --git a/funnel/assets/js/utils/prism.js b/funnel/assets/js/utils/prism.js new file mode 100644 index 000000000..089ef9cfc --- /dev/null +++ b/funnel/assets/js/utils/prism.js @@ -0,0 +1,52 @@ +const PrismEmbed = { + activatePrism() { + const activator = window.Prism.hooks.all.complete[0] || null; + if (activator) { + $('code[class*=language-]:not(.activated)').each(function activate() { + const languages = this.className + .split(' ') + .filter((cls) => cls.startsWith('language-')); + const language = languages[0].replace('language-', '') || null; + if (language) { + activator({ element: this, language }); + this.classList.add('activated'); + } + }); + } + }, + loadPrism() { + const CDN_CSS = 'https://unpkg.com/prismjs/themes/prism.min.css'; + const CDN = [ + 'https://unpkg.com/prismjs/components/prism-core.min.js', + 'https://unpkg.com/prismjs/plugins/autoloader/prism-autoloader.min.js', + ]; + let asset = 0; + const loadPrismStyle = () => { + $('head').append($(``)); + }; + const loadPrismScript = () => { + $.ajax({ + url: CDN[asset], + dataType: 'script', + cache: true, + }).done(() => { + if (asset < CDN.length - 1) { + asset += 1; + loadPrismScript(); + } else this.activatePrism(); + }); + }; + if (!window.Prism) { + loadPrismStyle(); + loadPrismScript(); + } + }, + init(containerDiv) { + this.containerDiv = containerDiv; + if ($('code[class*=language-]:not(.activated)').length > 0) { + this.loadPrism(); + } + }, +}; + +export default PrismEmbed; From ad26b6107246f5e80408cdeeec5fed16801789f2 Mon Sep 17 00:00:00 2001 From: Vidya Ramakrishnan Date: Tue, 15 Nov 2022 11:39:51 +0530 Subject: [PATCH 32/72] Fix delete operation in labels and submission form page (#1516) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix delete * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Bump loader-utils from 2.0.2 to 2.0.3 (#1514) Bumps [loader-utils](https://github.com/webpack/loader-utils) from 2.0.2 to 2.0.3. - [Release notes](https://github.com/webpack/loader-utils/releases) - [Changelog](https://github.com/webpack/loader-utils/blob/v2.0.3/CHANGELOG.md) - [Commits](https://github.com/webpack/loader-utils/compare/v2.0.2...v2.0.3) --- updated-dependencies: - dependency-name: loader-utils dependency-type: indirect ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Fix breakage caused by Flask-SQLAlchemy 3.0 (#1511) * [pre-commit.ci] pre-commit autoupdate (#1518) updates: - [github.com/jazzband/pip-tools: 6.9.0 → 6.10.0](https://github.com/jazzband/pip-tools/compare/6.9.0...6.10.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Fix delete * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix review comments * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: dependabot[bot] Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Kiran Jonnalagadda --- funnel/assets/js/submission_form.js | 5 +- funnel/assets/js/utils/formhelper.js | 75 +++++++++++-------- .../templates/collaborator_list.html.jinja2 | 6 +- funnel/templates/labels.html.jinja2 | 4 +- funnel/views/label.py | 60 ++++++++++----- 5 files changed, 91 insertions(+), 59 deletions(-) diff --git a/funnel/assets/js/submission_form.js b/funnel/assets/js/submission_form.js index 9a589ada1..9dbe93db7 100644 --- a/funnel/assets/js/submission_form.js +++ b/funnel/assets/js/submission_form.js @@ -122,10 +122,7 @@ $(() => { ); }); - const onSuccessFn = (responseData) => { - updateCollaboratorsList(responseData, false); - }; - Form.handleDelete('.js-remove-collaborator', onSuccessFn); + Form.handleDelete('.js-remove-collaborator', updateCollaboratorsList); SortItem($('.js-collaborator-list'), 'collaborator-placeholder', sortUrl); }; diff --git a/funnel/assets/js/utils/formhelper.js b/funnel/assets/js/utils/formhelper.js index 98583e8a7..4cd2c7988 100644 --- a/funnel/assets/js/utils/formhelper.js +++ b/funnel/assets/js/utils/formhelper.js @@ -108,7 +108,7 @@ const Form = { handleDelete(elementClass, onSucessFn) { $('body').on('click', elementClass, async function remove(event) { event.preventDefault(); - const url = $(this).attr('href'); + const url = $(this).attr('data-href'); const confirmationText = window.gettext('Are you sure you want to remove %s?', [ $(this).attr('title'), ]); @@ -125,7 +125,7 @@ const Form = { }).toString(), }).catch(Form.handleFetchNetworkError); if (response && response.ok) { - const responseData = await response.text(); + const responseData = await response.json(); if (responseData) { onSucessFn(responseData); } @@ -136,35 +136,50 @@ const Form = { }); }, activateToggleSwitch(callbckfn = '') { - $('body').on('change', '.js-toggle', function submitToggleSwitch() { - const checkbox = $(this); - const currentState = this.checked; - const previousState = !currentState; - const formData = new FormData($(checkbox).parent('form')[0]); - if (!currentState) { - formData.append($(this).attr('name'), false); - } - - fetch($(checkbox).parent('form').attr('action'), { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/x-www-form-urlencoded', - }, - body: new URLSearchParams(formData).toString(), - }) - .then((responseData) => { - if (responseData && responseData.message) { - window.toastr.success(responseData.message); + function postForm() { + let submitting = false; + return (checkboxElem) => { + if (!submitting) { + submitting = true; + const checkbox = $(checkboxElem); + const currentState = checkboxElem.checked; + const previousState = !currentState; + const formData = new FormData(checkbox.parent('form')[0]); + if (!currentState) { + formData.append(checkbox.attr('name'), false); } - if (callbckfn) { - callbckfn(); - } - }) - .catch((error) => { - Form.handleAjaxError(error); - $(checkbox).prop('checked', previousState); - }); + + fetch(checkbox.parent('form').attr('action'), { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: new URLSearchParams(formData).toString(), + }) + .then((responseData) => { + if (responseData && responseData.message) { + window.toastr.success(responseData.message); + } + if (callbckfn) { + callbckfn(); + } + submitting = false; + }) + .catch((error) => { + console.log(error); + Form.handleAjaxError(error); + checkbox.prop('checked', previousState); + submitting = false; + }); + } + }; + } + + const throttleSubmit = postForm(); + + $('body').on('change', '.js-toggle', function submitToggleSwitch() { + throttleSubmit(this); }); $('body').on('click', '.js-dropdown-toggle', function stopPropagation(event) { diff --git a/funnel/templates/collaborator_list.html.jinja2 b/funnel/templates/collaborator_list.html.jinja2 index 8af3514fe..6f1d61a8c 100644 --- a/funnel/templates/collaborator_list.html.jinja2 +++ b/funnel/templates/collaborator_list.html.jinja2 @@ -25,9 +25,9 @@
-
diff --git a/funnel/templates/labels.html.jinja2 b/funnel/templates/labels.html.jinja2 index f5a3bd40d..b9720d570 100644 --- a/funnel/templates/labels.html.jinja2 +++ b/funnel/templates/labels.html.jinja2 @@ -47,9 +47,9 @@
{{ faicon(icon='edit', icon_size='title') }} {% if label.has_proposals %} - {{ faicon(icon='archive', icon_size='title') }} + {{ faicon(icon='archive', icon_size='title') }} {% else %} - {{ faicon(icon='trash-alt', icon_size='title') }} + {{ faicon(icon='trash-alt', icon_size='title') }} {% endif %}
diff --git a/funnel/views/label.py b/funnel/views/label.py index 6eff22c31..bd6d5806c 100644 --- a/funnel/views/label.py +++ b/funnel/views/label.py @@ -8,6 +8,7 @@ from werkzeug.datastructures import MultiDict from baseframe import _, forms +from baseframe.forms import render_form from coaster.views import ModelView, UrlForView, render_with, requires_roles, route from .. import app @@ -204,8 +205,8 @@ def edit(self) -> ReturnRenderWith: 'project': self.obj.project, } - @route('archive', methods=['POST']) - @requires_login + @route('archive', methods=['GET', 'POST']) + @requires_sudo @requires_roles({'project_editor'}) def archive(self) -> ReturnView: form = forms.Form() @@ -213,31 +214,50 @@ def archive(self) -> ReturnView: self.obj.archived = True db.session.commit() flash(_("The label has been archived"), category='success') - else: - flash(_("CSRF token is missing"), category='error') - return render_redirect(self.obj.project.url_for('labels')) + return render_redirect(self.obj.project.url_for('labels')) + return render_form( + form=form, + title=_("Confirm archive of label"), + message=_("Archive this label?"), + submit=_("Archive"), + cancel_url=self.obj.project.url_for('labels'), + ) @route('delete', methods=['GET', 'POST']) @requires_sudo @requires_roles({'project_editor'}) def delete(self) -> ReturnView: - if self.obj.has_proposals: - flash( - _("Labels that have been assigned to submissions cannot be deleted"), - category='error', - ) - else: - if self.obj.has_options: - for olabel in self.obj.options: - db.session.delete(olabel) - db.session.delete(self.obj) - db.session.commit() + form = forms.Form() - if self.obj.main_label: - self.obj.main_label.options.reorder() + if form.validate_on_submit(): + if self.obj.has_proposals: + flash( + _( + "Labels that have been assigned to submissions cannot be deleted" + ), + category='error', + ) + else: + if self.obj.has_options: + for olabel in self.obj.options: + db.session.delete(olabel) + db.session.delete(self.obj) db.session.commit() - flash(_("The label has been deleted"), category='success') - return render_redirect(self.obj.project.url_for('labels')) + + if self.obj.main_label: + self.obj.main_label.options.reorder() + db.session.commit() + flash(_("The label has been deleted"), category='success') + return render_redirect(self.obj.project.url_for('labels')) + return render_form( + form=form, + title=_("Confirm delete"), + message=_( + "Delete this label? This operation is permanent and cannot be undone" + ), + submit=_("Delete"), + cancel_url=self.obj.project.url_for('labels'), + ) LabelView.init_app(app) From 02ed3d79bf5255e94708b3f08ae13969c46377a4 Mon Sep 17 00:00:00 2001 From: Kiran Jonnalagadda Date: Tue, 15 Nov 2022 15:02:20 +0530 Subject: [PATCH 33/72] Add explicit timeout for all requests (#1521) --- funnel/cli/periodic.py | 2 +- funnel/extapi/boxoffice.py | 2 +- funnel/extapi/explara.py | 5 ++++- funnel/forms/account.py | 2 +- funnel/loginproviders/github.py | 1 + funnel/loginproviders/linkedin.py | 1 + funnel/loginproviders/zoom.py | 1 + funnel/transports/sms/send.py | 1 + 8 files changed, 11 insertions(+), 4 deletions(-) diff --git a/funnel/cli/periodic.py b/funnel/cli/periodic.py index 522ffa066..2682e6d91 100644 --- a/funnel/cli/periodic.py +++ b/funnel/cli/periodic.py @@ -249,12 +249,12 @@ def trend_symbol(current, previous): requests.post( f'https://api.telegram.org/bot{app.config["TELEGRAM_STATS_APIKEY"]}' f'/sendMessage', + timeout=30, data={ 'chat_id': app.config['TELEGRAM_STATS_CHATID'], 'parse_mode': 'markdown', 'text': message, }, - timeout=30, ) diff --git a/funnel/extapi/boxoffice.py b/funnel/extapi/boxoffice.py index 7f4975ac7..1f3300e55 100644 --- a/funnel/extapi/boxoffice.py +++ b/funnel/extapi/boxoffice.py @@ -28,7 +28,7 @@ def get_orders(self, ic): self.base_url, f'ic/{ic}/orders?access_token={self.access_token}', ) - return requests.get(url).json().get('orders') + return requests.get(url, timeout=30).json().get('orders') def get_tickets(self, ic): tickets = [] diff --git a/funnel/extapi/explara.py b/funnel/extapi/explara.py index 10606a2d4..457fcfbac 100644 --- a/funnel/extapi/explara.py +++ b/funnel/extapi/explara.py @@ -47,7 +47,10 @@ def get_orders(self, explara_eventid): 'toRecord': to_record, } attendee_response = requests.post( - self.url_for('attendee-list'), headers=self.headers, data=payload + self.url_for('attendee-list'), + timeout=30, + headers=self.headers, + data=payload, ).json() if not attendee_response.get('attendee'): completed = True diff --git a/funnel/forms/account.py b/funnel/forms/account.py index c5ed2dcf6..71f195af6 100644 --- a/funnel/forms/account.py +++ b/funnel/forms/account.py @@ -114,7 +114,7 @@ def pwned_password_validator(_form, field) -> None: prefix, suffix = phash[:5], phash[5:] try: - rv = requests.get(f'https://api.pwnedpasswords.com/range/{prefix}') + rv = requests.get(f'https://api.pwnedpasswords.com/range/{prefix}', timeout=10) if rv.status_code != 200: # API call had an error and we can't proceed with validation. return diff --git a/funnel/loginproviders/github.py b/funnel/loginproviders/github.py index bbb6eca67..7feac9be2 100644 --- a/funnel/loginproviders/github.py +++ b/funnel/loginproviders/github.py @@ -49,6 +49,7 @@ def callback(self) -> LoginProviderData: try: response = requests.post( self.token_url, + timeout=30, headers={'Accept': 'application/json'}, params={ 'client_id': self.key, diff --git a/funnel/loginproviders/linkedin.py b/funnel/loginproviders/linkedin.py index 08a548afb..97b499f98 100644 --- a/funnel/loginproviders/linkedin.py +++ b/funnel/loginproviders/linkedin.py @@ -65,6 +65,7 @@ def callback(self) -> LoginProviderData: try: response = requests.post( self.token_url, + timeout=30, headers={'Accept': 'application/json'}, params={ 'grant_type': 'authorization_code', diff --git a/funnel/loginproviders/zoom.py b/funnel/loginproviders/zoom.py index f4158a4ae..290dc0548 100644 --- a/funnel/loginproviders/zoom.py +++ b/funnel/loginproviders/zoom.py @@ -53,6 +53,7 @@ def callback(self) -> LoginProviderData: try: response = requests.post( self.token_url, + timeout=30, headers={ 'Accept': 'application/x-www-form-urlencoded', 'Authorization': 'Basic ' diff --git a/funnel/transports/sms/send.py b/funnel/transports/sms/send.py index cd679ae7c..6dffed12d 100644 --- a/funnel/transports/sms/send.py +++ b/funnel/transports/sms/send.py @@ -104,6 +104,7 @@ def send_via_exotel(phone: str, message: SmsTemplate, callback: bool = True) -> try: r = requests.post( f'https://twilix.exotel.in/v1/Accounts/{sid}/Sms/send.json', + timeout=30, auth=(sid, token), data=payload, ) From 7c5c831ad1dd4b075bfbb24a3f4455af6d1eb024 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 25 Nov 2022 17:25:22 +0530 Subject: [PATCH 34/72] Disable the strikethrough token for markdown's basic profile --- funnel/utils/markdown/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 913321027..149d1ae1c 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -55,7 +55,7 @@ class MarkdownProfile: class MarkdownProfileBasic(MarkdownProfile): - post_config: PostConfig = {'disable': ['table']} + post_config: PostConfig = {'disable': ['table', 'strikethrough']} class MarkdownProfileDocument(MarkdownProfile): From 823b1876250feaa027c3387cc0c2fbec39e69189 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 25 Nov 2022 18:41:51 +0530 Subject: [PATCH 35/72] Changes in profiles. Basic is now just commonmark. Document is GFM plus our plugins. Text field only has support for strong, emphasis and single backticks. --- funnel/utils/markdown/profiles.py | 37 +++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/funnel/utils/markdown/profiles.py b/funnel/utils/markdown/profiles.py index 7f6a3159e..42493917e 100644 --- a/funnel/utils/markdown/profiles.py +++ b/funnel/utils/markdown/profiles.py @@ -16,7 +16,7 @@ sup_plugin, ) -__all__ = ['profiles', 'plugins', 'plugin_configs', 'default_markdown_options'] +__all__ = ['profiles', 'plugins', 'plugin_configs'] plugins: Dict[str, Callable] = { 'footnote': footnote.footnote_plugin, @@ -46,14 +46,6 @@ } -default_markdown_options = { - 'html': False, - 'linkify': True, - 'typographer': True, - 'breaks': True, -} - - class PostConfig(TypedDict): disable: NotRequired[List[str]] enable: NotRequired[List[str]] @@ -61,17 +53,32 @@ class PostConfig(TypedDict): class MarkdownProfile: - args: Tuple[str, Mapping] = ('gfm-like', default_markdown_options) + args: Tuple[str, Mapping] = ( + 'commonmark', + { + 'html': False, + 'breaks': True, + }, + ) plugins: List[str] = [] post_config: PostConfig = {} render_with: str = 'render' class MarkdownProfileBasic(MarkdownProfile): - post_config: PostConfig = {'disable': ['table', 'strikethrough']} + pass class MarkdownProfileDocument(MarkdownProfile): + args: Tuple[str, Mapping] = ( + 'gfm-like', + { + 'html': False, + 'linkify': True, + 'typographer': True, + 'breaks': True, + }, + ) plugins: List[str] = [ 'footnote', 'heading_anchors', @@ -85,10 +92,16 @@ class MarkdownProfileDocument(MarkdownProfile): 'vega-lite', 'mermaid', ] + post_config: PostConfig = {'enable': ['smartquotes']} class MarkdownProfileTextField(MarkdownProfile): - args: Tuple[str, Mapping] = ('zero', default_markdown_options) + args: Tuple[str, Mapping] = ( + 'zero', + { + 'html': False, + }, + ) post_config: PostConfig = { 'enable': [ 'emphasis', From d30adfc2a4acec60080e0b74001f64b5af1192a7 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 25 Nov 2022 18:54:10 +0530 Subject: [PATCH 36/72] Updated expected output of markdown tests, subsequent to changes in profile definitions. --- tests/data/markdown/basic.toml | 12 +++++------ tests/data/markdown/basic_plugins.toml | 28 +++++++++++++------------- tests/data/markdown/code.toml | 2 +- tests/data/markdown/headings.toml | 4 ++-- tests/data/markdown/links.toml | 12 +++++------ 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/data/markdown/basic.toml b/tests/data/markdown/basic.toml index 01a722dc6..2946c0cab 100644 --- a/tests/data/markdown/basic.toml +++ b/tests/data/markdown/basic.toml @@ -80,7 +80,7 @@ profiles = [ "basic", "document",] [expected_output] basic = """

Basic markup

This is a sample paragraph that has asterisk bold, asterisk emphasized, underscore bold and underscore italic strings.

-

This is a finalsample paragraph that has an asterisk bold italic string and an underscore bold italic string.
+

This is a ~~final~~sample paragraph that has an asterisk bold italic string and an underscore bold italic string.
It also has a newline break here!!!

Here are examples of bold and emphasized text depending on the placement of underscores/asterisks:

__Bold without closure does not work
@@ -90,7 +90,7 @@ _Emphasis without closure does not work

Bold without closure
on the same line
carries forward to
-this is text that should strike off
+~~this is text that should strike off~~
multiple consecutive lines

Bold without closure
on the same line
@@ -107,7 +107,7 @@ multiple consecutive lines

Horizontal Rules

Above is a horizontal rule using hyphens.
-This is text that should strike off

+~~This is text that should strike off~~

Above is a horizontal rule using asterisks.
Below is a horizontal rule using underscores.

@@ -115,7 +115,7 @@ Below is a horizontal rule using underscores.

Links

Here is a link to hasgeek

Link to funnel with the title 'Hasgeek'

-

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

+

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

Markdown-it typography

The results of the below depend on the typographer options enabled by us in the markdown-it-py parser, if typographer=True has been passed to it.

@@ -174,7 +174,7 @@ Below is a horizontal rule using underscores.

Here is a link to hasgeek

-

Link to funnel with the title 'Hasgeek'

+

Link to funnel with the title ‘Hasgeek’

Autoconverted link https://github.com/hasgeek (will autoconvert if linkify is on)

Markdown-it typography

@@ -184,7 +184,7 @@ Below is a horizontal rule using underscores.

test.. test... test..... test?..... test!....
!!!!!! ???? ,, -- ---

The below should convert the quotes if smartquotes has been enabled.

-

"Smartypants, double quotes" and 'single quotes'

+

“Smartypants, double quotes” and ‘single quotes’

Blockquotes

diff --git a/tests/data/markdown/basic_plugins.toml b/tests/data/markdown/basic_plugins.toml index 33ed6b3ae..06e612b99 100644 --- a/tests/data/markdown/basic_plugins.toml +++ b/tests/data/markdown/basic_plugins.toml @@ -98,13 +98,13 @@ plugins = [ "ins","del","sup","sub","mark"] [expected_output] basic = """

Testing markdown-it-py plugins for ins/del/sup/sub/mark

-

This is ++inserted text++and deleted text.

+

This is ++inserted text++~~and deleted text~~.

This is ^superscript^, this is ~subscript~ and this is ==marked text==.

This is not +inserted text+, nor is this ~deleted text~.

^Super script^ and ~sub script~, do not support whitespaces.

++This is a
-multiline insert++ and a
-multiline delete

+multiline insert++ ~~and a
+multiline delete~~

<pre>
     <code>
         This should not render as deleted text, rather it should be a fenced block.
@@ -116,7 +116,7 @@ multiline delete

 This is the middle of the paragraph.

 ++These lines have

 also been inserted in it.++

-While this line has been deleted.

+~~While this line has been deleted.~~

 The paragraph ends here.

 
This is +++an insert++ed text+++

 This is ++++also an insert++ed text++++

@@ -124,13 +124,13 @@ This is ++++also an insert++ed text++++

 
This text has ++an inserted text ++with another inserted text++ ++with another inserted text++++

 
+Hello World! I cannot assert myself as an inserted text.+

 ++foobar()++

-
This is ~a delet~~ed text.~

-This is also a delet~~ed text

-
This text has a deleted text with another deleted text

-
This text has a deleted text with another deleted text with another inserted text

+
This is ~~~a delet~~ed text.~~~

+This is ~~~~also a delet~~ed text~~~~

+
This text has ~~a deleted text ~~with another deleted text~~~~

+
This text has ~~a deleted text ~~with another deleted text~~ ~~with another inserted text~~~~

 
~Hello World! I cannot assert myself as an inserted text nor a subscript.~

-foobar()

-
Now, we will list out some fruits++grocery++:

+~~foobar()~~

+
Now, we will list out some ~~fruits~~++grocery++:

 
    
 
  • ++Fruits++
 
      
@@ -151,11 +151,11 @@ This is also a delet~~ed text
      
 bold ++emphasized insert++
      
 bold++insert emphasized++
      
 bold_++emphasized insert++_
      
-
      bolddelete
      
-bold delete emphasized
      
-bold emphasized delete
      
+
      bold~~delete~~
      
+bold ~~delete emphasized~~
      
+bold ~~emphasized delete~~
      
 bold~~delete emphasized~~
      
-bold*emphasized delete*
      
+bold*~~emphasized delete~~*
      
 
      bold^super^
      
 bold ^super-emphasized^
      
 bold ^emphasized-super^
      
diff --git a/tests/data/markdown/code.toml b/tests/data/markdown/code.toml
index 3ebee0aff..1ed8a4c3f 100644
--- a/tests/data/markdown/code.toml
+++ b/tests/data/markdown/code.toml
@@ -119,7 +119,7 @@ line 1 of code
 line 2 of code
 line 3 of code
-

Block code "fences"

+

Block code “fences” 

Sample text here...
 It is a sample text that has multiple lines
diff --git a/tests/data/markdown/headings.toml b/tests/data/markdown/headings.toml index b4c962431..31157a541 100644 --- a/tests/data/markdown/headings.toml +++ b/tests/data/markdown/headings.toml @@ -50,7 +50,7 @@ Only the specified headings in extension defaults should get linked.

######h6 Heading

Text with 2 or more hyphens below it converts to H2

""" -document = """

Using the heading-anchors plugin with it's default config:

+document = """

Using the heading-anchors plugin with it’s default config:

The below headings should convert and get linked.
Only the specified headings in extension defaults should get linked.

h1 Heading

@@ -69,7 +69,7 @@ Only the specified headings in extension defaults should get linked.

######h6 Heading

Text with 2 or more hyphens below it converts to H2

""" -heading_anchors = """

Using the heading-anchors plugin with it’s default config:

+heading_anchors = """

Using the heading-anchors plugin with it's default config:

The below headings should convert and get linked.
Only the specified headings in extension defaults should get linked.

h1 Heading

diff --git a/tests/data/markdown/links.toml b/tests/data/markdown/links.toml index b1760c7c2..fbe1aa363 100644 --- a/tests/data/markdown/links.toml +++ b/tests/data/markdown/links.toml @@ -67,8 +67,8 @@ basic = """

Links

Hasgeek
Hasgeek TV

Autoconverted links

-

https://github.com/nodeca/pica
-http://twitter.com/hasgeek

+

https://github.com/nodeca/pica
+http://twitter.com/hasgeek

Footnote style syntax

Hasgeek

Links can have a footnote style syntax, where the links can be defined later. This is helpful in the case of a repetitive URL that needs to be linked to.

@@ -91,7 +91,7 @@ basic = """

Links

[notmalicious](javascript://%0d%0awindow.onerror=alert;throw%20document.cookie)
[a](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
[clickme](vbscript:alert(document.domain))
-http://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
+http://danlec@.1 style=background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAABACAMAAADlCI9NAAACcFBMVEX/AAD//////f3//v7/0tL/AQH/cHD/Cwv/+/v/CQn/EBD/FRX/+Pj/ISH/PDz/6Oj/CAj/FBT/DAz/Bgb/rq7/p6f/gID/mpr/oaH/NTX/5+f/mZn/wcH/ICD/ERH/Skr/3Nz/AgL/trb/QED/z8//6+v/BAT/i4v/9fX/ZWX/x8f/aGj/ysr/8/P/UlL/8vL/T0//dXX/hIT/eXn/bGz/iIj/XV3/jo7/W1v/wMD/Hh7/+vr/t7f/1dX/HBz/zc3/nJz/4eH/Zmb/Hx//RET/Njb/jIz/f3//Ojr/w8P/Ghr/8PD/Jyf/mJj/AwP/srL/Cgr/1NT/5ub/PT3/fHz/Dw//eHj/ra3/IiL/DQ3//Pz/9/f/Ly//+fn/UFD/MTH/vb3/7Oz/pKT/1tb/2tr/jY3/6en/QkL/5OT/ubn/JSX/MjL/Kyv/Fxf/Rkb/sbH/39//iYn/q6v/qqr/Y2P/Li7/wsL/uLj/4+P/yMj/S0v/GRn/cnL/hob/l5f/s7P/Tk7/WVn/ior/09P/hYX/bW3/GBj/XFz/aWn/Q0P/vLz/KCj/kZH/5eX/U1P/Wlr/cXH/7+//Kir/r6//LS3/vr7/lpb/lZX/WFj/ODj/a2v/TU3/urr/tbX/np7/BQX/SUn/Bwf/4uL/d3f/ExP/y8v/NDT/KSn/goL/8fH/qan/paX/2Nj/HR3/4OD/VFT/Z2f/SEj/bm7/v7//RUX/Fhb/ycn/V1f/m5v/IyP/xMT/rKz/oKD/7e3/dHT/h4f/Pj7/b2//fn7/oqL/7u7/2dn/TEz/Gxv/6ur/3d3/Nzf/k5P/EhL/Dg7/o6P/UVHe/LWIAAADf0lEQVR4Xu3UY7MraRRH8b26g2Pbtn1t27Zt37Ft27Zt6yvNpPqpPp3GneSeqZo3z3r5T1XXL6nOFnc6nU6n0+l046tPruw/+Vil/C8tvfscquuuOGTPT2ZnRySwWaFQqGG8Y6j6Zzgggd0XChWLf/U1OFoQaVJ7AayUwPYALHEM6UCWBDYJbhXfHjUBOHvVqz8YABxfnDCArrED7jSAs13Px4Zo1jmA7eGEAXvXjRVQuQE4USWqp5pNoCthALePFfAQ0OcchoCGBAEPgPGiE7AiacChDfBmjjg7DVztAKRtnJsXALj/Hpiy2B9wofqW9AQAg8Bd8VOpCR02YMVEE4xli/L8AOmtQMQHsP9IGUBZedq/AWJfIez+x4KZqgDtBlbzon6A8GnonOwBXNONavlmUS2Dx8XTjcCwe1wNvGQB2gxaKhbV7Ubx3QC5bRMUuAEvA9kFzzW3TQAeVoB5cFw8zQUGPH9M4LwFgML5IpL6BHCvH0DmAD3xgIUpUJcTmy7UQHaV/bteKZ6GgGr3eAq4QQEmWlNqJ1z0BeTvgGfz4gAFsDXfUmbeAeoAF0OfuLL8C91jHnCtBchYq7YzsMsXIFkmDDsBjwBfi2o6GM9IrOshIp5mA6vc42Sg1wJMEVUJlPgDpBzWb3EAVsMOm5m7Hg5KrAjcJJ5uRn3uLAvosgBrRPUgnAgApC2HjtpRwFTneZRpqLs6Ak+Lp5lAj9+LccoCzLYPZjBA3gIGRgHj4EuxewH6JdZhKBVPM4CL7rEIiKo7kMAvILIEXplvA/bCR2JXAYMSawtkiqfaDHjNtYVfhzJJBvBGJ3zmADhv6054W71ZrBNvHZDigr0DDCcFkHeB8wog70G/2LXA+xIrh03i02Zgavx0Blo+SA5Q+yEcrVSAYvjYBhwEPrEoDZ+KX20wIe7G1ZtwTJIDyMYU+FwBeuGLpaLqg91NcqnqgQU9Yre/ETpzkwXIIKAAmRnQruboUeiVS1cHmF8pcv70bqBVkgak1tgAaYbuw9bj9kFjVN28wsJvxK9VFQDGzjVF7d9+9z1ARJIHyMxRQNo2SDn2408HBsY5njZJPcFbTomJo59H5HIAUmIDpPQXVGS0igfg7detBqptv/0ulwfIbbQB8kchVtNmiQsQUO7Qru37jpQX7WmS/6YZPXP+LPprbVgC0ul0Op1Op9Pp/gYrAa7fWhG7QQAAAABJRU5ErkJggg==);background-repeat:no-repeat;display:block;width:100%;height:100px; onclick=alert(unescape(/Oh%20No!/.source));return(false);//
text
[a](javascript:this;alert(1))
[a](javascript:this;alert(1))
@@ -110,8 +110,8 @@ basic = """

Links

a
</http://<?php><\\h1>script:scriptconfirm(2)
XSS
-[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
-[ ](http://a?p=[[/onclick=alert(0) .]])
+[ ](https://a.de?p=[[/data-x=. style=background-color:#000000;z-index:999;width:100%;position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])
+[ ](http://a?p=[[/onclick=alert(0) .]])
[a](javascript:new%20Function`al\\ert`1``;)

""" document = """ @@ -133,7 +133,7 @@ document = """