From 6fc9b9b63bdd8600f475148da3b035152fab7ec2 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Sun, 8 Sep 2019 15:08:09 +0200 Subject: [PATCH 01/10] chore(docs): Gitignore generated files --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 219774b01..15d1ae7a8 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,8 @@ src/cozy-realtime # docs/ is generated, no need to have it in the repo docs/ + +*.pyc +*.egg-info +.DS_Store +*.egg From fab67dbd8d2deb034180c446aa2f8ea6a5d81f78 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Sun, 8 Sep 2019 01:09:26 +0200 Subject: [PATCH 02/10] feat: Bridge between pymarkdown and SphinxJS Implemented as a pymarkdown extension using the gather_doclets function from Sphinx --- markdown_sphinxjs/README.md | 8 + .../markdown_sphinxjs/__init__.py | 151 ++++++++++++++++++ markdown_sphinxjs/requirements.txt | 1 + markdown_sphinxjs/setup.py | 114 +++++++++++++ markdown_sphinxjs/test.py | 20 +++ src/css/extra.css | 10 +- 6 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 markdown_sphinxjs/README.md create mode 100644 markdown_sphinxjs/markdown_sphinxjs/__init__.py create mode 100644 markdown_sphinxjs/requirements.txt create mode 100755 markdown_sphinxjs/setup.py create mode 100644 markdown_sphinxjs/test.py diff --git a/markdown_sphinxjs/README.md b/markdown_sphinxjs/README.md new file mode 100644 index 000000000..27658b55e --- /dev/null +++ b/markdown_sphinxjs/README.md @@ -0,0 +1,8 @@ +Markdown SphinxJS +================= + +Auto import JSDoc in your Markdown documentation. + +This is based on the wonderful [SphinxJS] that does the same for the Sphinx documentation generator. + +[SphinxJS]: https://github.com/mozilla/sphinx-js/ diff --git a/markdown_sphinxjs/markdown_sphinxjs/__init__.py b/markdown_sphinxjs/markdown_sphinxjs/__init__.py new file mode 100644 index 000000000..0c632bcae --- /dev/null +++ b/markdown_sphinxjs/markdown_sphinxjs/__init__.py @@ -0,0 +1,151 @@ +from __future__ import absolute_import +from __future__ import unicode_literals +from markdown.extensions import Extension +from markdown.blockprocessors import BlockProcessor +from markdown.util import etree +import re +import os +import json + +from sphinx_js import gather_doclets + + +class Config: + def __init__(self, **attrs): + for key, val in attrs.items(): + setattr(self, key, val) + + +class App: + """Make alike for Sphinx app for SphinxJS to work""" + + def __init__(self, config): + self.config = Config(**config) + self._sphinxjs_doclets_by_class = {} + self.confdir = "/tmp" + + +def gather_doclets_from_dir(src_dir): + app = App({ + 'js_source_path': src_dir, + 'js_language': 'javascript', + 'root_for_relative_js_paths': src_dir, + 'jsdoc_config_path': None + }) + gather_doclets(app) + return { + "by_class": app._sphinxjs_doclets_by_class, + "by_path": app._sphinxjs_doclets_by_path, + } + + +def make_definition_node(ancestor, definition, path): + div = etree.SubElement(ancestor, "div") + div.attrib["class"] = "markdown-sphinxjs-description" + + name = etree.SubElement(div, "h4") + name.text = "%s.%s(%s) => %s" % ( + definition["memberof"], + definition["name"], + ", ".join(definition["meta"]["code"]["paramnames"]), + definition["returns"][0]["type"]["names"][0] + ) + p = etree.SubElement(div, "p") + p.text = definition["description"] + param_table = etree.SubElement(div, "table") + param_head = etree.SubElement(param_table, "thead") + head_row = etree.SubElement(param_table, "tr") + name = etree.SubElement(head_row, "th") + name.text = 'Parameter' + type = etree.SubElement(head_row, "th") + type.text = 'Type' + desc = etree.SubElement(head_row, "th") + desc.text = 'Description' + + # data = etree.SubElement(div, "pre") + # data.text = json.dumps(definition, indent=2) + + params = etree.SubElement(param_table, "tbody") + for param in definition["params"]: + row = etree.SubElement(params, "tr") + name = etree.SubElement(row, "td") + name.text = param["name"] + type = etree.SubElement(row, "td") + type.text = ", ".join(param["type"]["names"]) + desc = etree.SubElement(row, "td") + desc.text = param["description"] + + for example in definition["examples"]: + example_node = etree.SubElement(div, "pre") + example_node.text = """%s""" % example + + return div + + +class MarkdownJSExtension(Extension): + def __init__(self, directory, **kwargs): + super(MarkdownJSExtension, self).__init__(**kwargs) + self.config = {"directory": directory} + self.index = {} + self.doclets = gather_doclets_from_dir(directory) + + def extendMarkdown(self, md, **kwargs): + md.registerExtension(self) + + md.parser.blockprocessors.register( + MarkdownJSProcessor(md.parser, self.doclets), "markdown-sphinxjs", 105 + ) + + +class MarkdownJSProcessor(BlockProcessor): + """ + Understands blocks beginining by ---> + The arrow must be followed by an identifier for a function. + + Finds the function referenced by identifier and outputs a div with the description + and parameters of the function. + + Mostly copied from admonition block processor""" + + RE = re.compile(r'^---> ?([\w\-/#]+(?: +[\w\-#]+)*)(?:\n|$)') + RE_SPACES = re.compile(" +") + + def __init__(self, parser, doclets): + super(MarkdownJSProcessor, self).__init__(parser) + self.doclets = doclets + + def test(self, parent, block): + sibling = self.lastChild(parent) + return self.RE.search(block) + + def build(self, ancestor, match): + path_tokens = match.group(1).split(' ') + definition, path = self.doclets["by_path"].get_with_path(path_tokens) + return make_definition_node(ancestor, definition, path) + + def run(self, parent, blocks): + sibling = self.lastChild(parent) + block = blocks.pop(0) + m = self.RE.search(block) + + if m: + block = block[m.end() :] # removes the first line + + block, theRest = self.detab(block) + + if m: + div = self.build(parent, m) + else: + div = sibling + + self.parser.parseChunk(div, block) + + if theRest: + # This block contained unindented line(s) after the first indented + # line. Insert these lines as the first block of the master blocks + # list for future processing. + blocks.insert(0, theRest) + + +def makeExtension(**kwargs): # pragma: no cover + return MarkdownJSExtension(**kwargs) diff --git a/markdown_sphinxjs/requirements.txt b/markdown_sphinxjs/requirements.txt new file mode 100644 index 000000000..56e80423b --- /dev/null +++ b/markdown_sphinxjs/requirements.txt @@ -0,0 +1 @@ +markdown==3.1.1 diff --git a/markdown_sphinxjs/setup.py b/markdown_sphinxjs/setup.py new file mode 100755 index 000000000..a6070df0a --- /dev/null +++ b/markdown_sphinxjs/setup.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Note: To use the 'upload' functionality of this file, you must: +# $ pip install twine + +import io +import os +import sys +from shutil import rmtree + +from setuptools import find_packages, setup, Command + +# Package meta-data. +NAME = 'markdownsphinxjs' +DESCRIPTION = 'SphinxJS for markdown' +URL = 'https://github.com/cozy/markdown_sphinxjs' +EMAIL = 'contact@cozycloud.cc' +AUTHOR = 'ptbrowne' +REQUIRES_PYTHON = '>=3.4.3' +VERSION = '1.0.0' + +# What packages are required for this module to be executed? +REQUIRED = ['markdown', 'sphinx_js'] + +# The rest you shouldn't have to touch too much :) +# ------------------------------------------------ +# Except, perhaps the License and Trove Classifiers! +# If you do change the License, remember to change the Trove Classifier for that! + +here = os.path.abspath(os.path.dirname(__file__)) + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.md' is present in your MANIFEST.in file! +with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = '\n' + f.read() + +# Load the package's __version__.py module as a dictionary. +about = {} +if not VERSION: + with open(os.path.join(here, NAME, '__version__.py')) as f: + exec(f.read(), about) +else: + about['__version__'] = VERSION + + +class UploadCommand(Command): + """Support setup.py upload.""" + + description = 'Build and publish the package.' + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print('\033[1m{0}\033[0m'.format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status('Removing previous builds…') + rmtree(os.path.join(here, 'dist')) + except OSError: + pass + + self.status('Building Source and Wheel (universal) distribution…') + os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable)) + + self.status('Uploading the package to PyPi via Twine…') + os.system('twine upload dist/*') + + self.status('Pushing git tags…') + os.system('git tag v{0}'.format(about['__version__'])) + os.system('git push --tags') + + sys.exit() + +print(find_packages(exclude=('tests',))) +# Where the magic happens: +setup( + name=NAME, + version=about['__version__'], + description=DESCRIPTION, + long_description=long_description, + long_description_content_type='text/markdown', + author=AUTHOR, + author_email=EMAIL, + python_requires=REQUIRES_PYTHON, + url=URL, + packages=find_packages(exclude=('tests',)), + # If your package is a single module, use this instead of 'packages': + #py_modules=['markdownsphinxjs'], + #entry_points={'console_scripts': ['mint=mint.cli:main']}, + install_requires=REQUIRED, + include_package_data=True, + license='MIT', + classifiers=[ + # Trove classifiers + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + ], + # $ setup.py publish support. + cmdclass={'upload': UploadCommand}, +) diff --git a/markdown_sphinxjs/test.py b/markdown_sphinxjs/test.py new file mode 100644 index 000000000..0800cc301 --- /dev/null +++ b/markdown_sphinxjs/test.py @@ -0,0 +1,20 @@ +import markdown +from jsdoc_reference import JSDocReferenceExtension + +ext = JSDocReferenceExtension(directory='/Users/cozy/code/cozy/konnector-libs/packages/cozy-konnector-libs/src') +m = markdown.Markdown(extensions=[ext]) + +html = m.convert( + """ +## Hello + +How are you ? + +--->findDuplicates + + +!!!!note + salut +""" +) +print(html) diff --git a/src/css/extra.css b/src/css/extra.css index 9960959b8..8147c0c1f 100644 --- a/src/css/extra.css +++ b/src/css/extra.css @@ -39,4 +39,12 @@ /* stack pattern */ .home-action a > * + * { margin-bottom: 1rem -} +} + +/* From markdown-sphinxjs */ +.markdown-sphinxjs-description { + border-top-left-radius: 2px; + border-bottom-left-radius: 2px; + border-left: 0.25rem #ccc solid; + padding-left: 1rem; +} From 123c05bd7f8ee54e97b37273410cd00dc0b36f57 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Sun, 8 Sep 2019 15:01:51 +0200 Subject: [PATCH 03/10] feat: Default config for JSDoc enables recursion by default --- .../markdown_sphinxjs/__init__.py | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/markdown_sphinxjs/markdown_sphinxjs/__init__.py b/markdown_sphinxjs/markdown_sphinxjs/__init__.py index 0c632bcae..77ca20b08 100644 --- a/markdown_sphinxjs/markdown_sphinxjs/__init__.py +++ b/markdown_sphinxjs/markdown_sphinxjs/__init__.py @@ -1,11 +1,13 @@ from __future__ import absolute_import from __future__ import unicode_literals -from markdown.extensions import Extension -from markdown.blockprocessors import BlockProcessor -from markdown.util import etree import re import os import json +import tempfile + +from markdown.extensions import Extension +from markdown.blockprocessors import BlockProcessor +from markdown.util import etree from sphinx_js import gather_doclets @@ -25,14 +27,39 @@ def __init__(self, config): self.confdir = "/tmp" -def gather_doclets_from_dir(src_dir): - app = App({ - 'js_source_path': src_dir, - 'js_language': 'javascript', - 'root_for_relative_js_paths': src_dir, - 'jsdoc_config_path': None - }) - gather_doclets(app) +DEFAULT_JSDOC_CONFIG = { + "opts": { + "recurse": True + }, + "source": { + "includePattern": ".+\\.js(doc)?x?$", + "excludePattern": "((^|\\/|\\\\)_)|(min)|(dist)", + "exclude": [ + "node_modules", + "plugins" + ] + } +} + + +def gather_doclets_from_dir(src_dir, jsdoc_cache=None, force=False): + if force and os.path.isfile(jsdoc_cache): + os.unlink(jsdoc_cache) + + with tempfile.NamedTemporaryFile(mode='w', delete=False) as configfile: + configfile.write(json.dumps(DEFAULT_JSDOC_CONFIG, indent=2)) + configfile.seek(0) + app = App( + { + "js_source_path": src_dir, + "js_language": "javascript", + "root_for_relative_js_paths": src_dir, + "jsdoc_config_path": configfile.name, + "jsdoc_cache": jsdoc_cache, + "sphinx_js_lax": True + } + ) + gather_doclets(app) return { "by_class": app._sphinxjs_doclets_by_class, "by_path": app._sphinxjs_doclets_by_path, From de3346d5e5406325b112535c0d90a310dc9d4fb7 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Sun, 8 Sep 2019 15:02:26 +0200 Subject: [PATCH 04/10] feat(extension): Doclets are stored in Extension --- markdown_sphinxjs/markdown_sphinxjs/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/markdown_sphinxjs/markdown_sphinxjs/__init__.py b/markdown_sphinxjs/markdown_sphinxjs/__init__.py index 77ca20b08..62bb01dfb 100644 --- a/markdown_sphinxjs/markdown_sphinxjs/__init__.py +++ b/markdown_sphinxjs/markdown_sphinxjs/__init__.py @@ -110,11 +110,18 @@ def make_definition_node(ancestor, definition, path): class MarkdownJSExtension(Extension): - def __init__(self, directory, **kwargs): + doclets = None + def __init__(self, directory, jsdoc_cache, **kwargs): super(MarkdownJSExtension, self).__init__(**kwargs) self.config = {"directory": directory} self.index = {} - self.doclets = gather_doclets_from_dir(directory) + if not MarkdownJSExtension.doclets: + # Markdown extensions are instantiated for each file processed but doclets + # gathering is costly and we do not want to do it every time, this is why + # we store the generated doclets in the class. + # This is a hacky way to have doclets computation be done only once. + MarkdownJSExtension.doclets = gather_doclets_from_dir(directory, jsdoc_cache) + self.doclets = MarkdownJSExtension.doclets def extendMarkdown(self, md, **kwargs): md.registerExtension(self) From aad62ec96532f74472413c898f0f1eabbb701ed5 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Sun, 8 Sep 2019 15:04:08 +0200 Subject: [PATCH 05/10] feat(config): Use argparse --- generate_config.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/generate_config.py b/generate_config.py index 6538ec0b0..3c1ea4009 100644 --- a/generate_config.py +++ b/generate_config.py @@ -3,6 +3,7 @@ - Generate mkdocs.yml from mkdocs.yml.tpl """ +import argparse import yaml import json import os @@ -184,10 +185,16 @@ def replace_toc_placeholders(nav, named_tocs): cur[single_key] = flatten_entry_if_single(toc) -def main(argv): + +def main(): OUTSIDE_DOCS = 'OUTSIDE_DOCS' - if '--fetch' in argv: + parser = argparse.ArgumentParser() + parser.add_argument('--fetch', action='store_true', help='Fetch all external repositories') + parser.add_argument('--force-jsdoc', action='store_true', help='Force re-analysis of jsdocs') + args = parser.parse_args() + + if args.fetch: fetch_all_external_docs_from_file(OUTSIDE_DOCS) with open('./mkdocs.yml.tpl') as f: @@ -219,4 +226,4 @@ def main(argv): warning = '#\n# THIS FILE IS AUTOMATICALLY GENERATED, PLEASE EDIT `mkdocs.yml.tpl` AND LAUNCH `python generate_config.py`\n#\n' f.write(warning + content) -main(sys.argv) +main() From ade033ada642b74ffa9ded1c854976dc3353f763 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Sun, 8 Sep 2019 00:50:07 +0200 Subject: [PATCH 06/10] refactor(config): Extract tmp dir functions --- generate_config.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/generate_config.py b/generate_config.py index 3c1ea4009..e74b12b72 100644 --- a/generate_config.py +++ b/generate_config.py @@ -74,6 +74,7 @@ def find_entry(tree, name): except IndexError: return None + def walk_dict(d, path=None): if path is None: path = [] @@ -115,16 +116,25 @@ def fetch_external_doc(repository, destination): sh.git('clone', repository, '--depth', '1', '.') +TMP_DIR = '/tmp/cozy_docs' +def ensure_root_tmp_dir(): + sh.mkdir('-p', TMP_DIR) + +def get_doc_tmp_dir(doc_name): + return osp.join(TMP_DIR, doc_name) + + def fetch_all_external_docs_from_file(filename): + ensure_root_tmp_dir() with open(filename) as f: external_docs = [parse_external_doc_line(l) for l in f] - for name, repository, doc_directory in external_docs: - tmpdir = osp.join('/tmp', name) - print('Fetching %s...' % name) + for docname, repository, doc_directory in external_docs: + print('Fetching %s...' % docname) + tmpdir = get_doc_tmp_dir(docname) fetch_external_doc(repository, tmpdir) - src_dir = osp.join('src', name) + src_dir = osp.join('src', docname) sh.rm('-f', src_dir) - print('Linking %s...' % name) + print('Linking %s...' % docname) sh.ln('-s', osp.join(tmpdir, doc_directory), src_dir) From ed1ac7857737ea605b3e97999cd1bc4c81ebe38c Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Sun, 8 Sep 2019 15:05:20 +0200 Subject: [PATCH 07/10] feat(config): Cache JSDoc data --- generate_config.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/generate_config.py b/generate_config.py index e74b12b72..ab9b805a9 100644 --- a/generate_config.py +++ b/generate_config.py @@ -13,6 +13,8 @@ from collections import OrderedDict, namedtuple import fnmatch import sh +from markdown_sphinxjs import gather_doclets_from_dir + def simple_glob(directory, glob_pattern): matches = [] @@ -120,6 +122,7 @@ def fetch_external_doc(repository, destination): def ensure_root_tmp_dir(): sh.mkdir('-p', TMP_DIR) + def get_doc_tmp_dir(doc_name): return osp.join(TMP_DIR, doc_name) @@ -195,6 +198,10 @@ def replace_toc_placeholders(nav, named_tocs): cur[single_key] = flatten_entry_if_single(toc) +def ensure_jsdoc_cache_exists(src_dir, force=False): + cache_file = '/tmp/jsdoc_data_cache.json' + gather_doclets_from_dir(src_dir, jsdoc_cache=cache_file, force=force) + def main(): OUTSIDE_DOCS = 'OUTSIDE_DOCS' @@ -207,6 +214,8 @@ def main(): if args.fetch: fetch_all_external_docs_from_file(OUTSIDE_DOCS) + ensure_jsdoc_cache_exists(TMP_DIR, force=args.force_jsdoc) + with open('./mkdocs.yml.tpl') as f: data = ordered_load(f, yaml.SafeLoader) From b223be3e215044f2c0d8a9f52bfffcabb03fe7fd Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Sun, 8 Sep 2019 15:06:43 +0200 Subject: [PATCH 08/10] feat(docs): Configure for markdownsphinx extension --- mkdocs.yml.tpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mkdocs.yml.tpl b/mkdocs.yml.tpl index c50d66cad..fe8733d7e 100644 --- a/mkdocs.yml.tpl +++ b/mkdocs.yml.tpl @@ -78,6 +78,9 @@ markdown_extensions: - smarty - toc: permalink: true +- markdown_sphinxjs: + directory: /tmp/cozy_docs + jsdoc_cache: /tmp/jsdoc_data_cache.json extra: search: tokenizer: "[^a-z\u0430-\u044F\u04510-9\\-\\.]" From f705d476ff974c16f562d1a508634e065f9ff46a Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Mon, 9 Sep 2019 09:00:25 +0200 Subject: [PATCH 09/10] feat: Add own fork of sphinxjs --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5798d74b2..39d2e5898 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,3 +29,4 @@ Unidecode==1.0.22 urllib3==1.23 mkdocs==1.0.4 mkdocs-material==4.4.0 +git+git://github.com/ptbrowne/sphinx-js@master#egg=sphinxjs From 68e475b1e7c8a99227d666aebbcdf935b5e47582 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Mon, 9 Sep 2019 01:02:09 +0200 Subject: [PATCH 10/10] chore: WIP Test file for markdown sphinx js --- src/tutorials/konnector/getting-started.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tutorials/konnector/getting-started.md b/src/tutorials/konnector/getting-started.md index 27fcf72c3..63a7164b5 100644 --- a/src/tutorials/konnector/getting-started.md +++ b/src/tutorials/konnector/getting-started.md @@ -7,6 +7,8 @@ The easiest way to create a new connector is to use [cozy-konnector-template](https://github.com/konnectors/cozy-konnector-template). +---> BaseKonnector# waitForTwoFaCode + First of all, [download](https://github.com/konnectors/cozy-konnector-template/archive/master.zip) or clone the repository: ```sh