build.py

#!/usr/bin/env python3

import argparse
import datetime
import logging
import os
import shutil
import subprocess
import sys
import time

import jinja2
import livereload
import markdown.util

import nav  # monkey patches mkdocs

from mkdocs import config
from mkdocs import exceptions
import mkdocs.commands.build

import amp
import blog
import mdx_clickhouse
import redirects
import single_page
import test
import util
import website

from cmake_in_clickhouse_generator import generate_cmake_flags_files


class ClickHouseMarkdown(markdown.extensions.Extension):
    class ClickHousePreprocessor(markdown.util.Processor):
        def run(self, lines):
            for line in lines:
                if "<!--hide-->" not in line:
                    yield line

    def extendMarkdown(self, md):
        md.preprocessors.register(
            self.ClickHousePreprocessor(), "clickhouse_preprocessor", 31
        )


markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown


def build_for_lang(lang, args):
    logging.info(f"Building {lang} docs")
    os.environ["SINGLE_PAGE"] = "0"

    try:
        theme_cfg = {
            "name": None,
            "custom_dir": os.path.join(os.path.dirname(__file__), "..", args.theme_dir),
            "language": lang,
            "direction": "rtl" if lang == "fa" else "ltr",
            "static_templates": ["404.html"],
            "extra": {
                "now": int(
                    time.mktime(datetime.datetime.now().timetuple())
                )  # TODO better way to avoid caching
            },
        }

        # the following list of languages is sorted according to
        # https://en.wikipedia.org/wiki/List_of_languages_by_total_number_of_speakers
        languages = {"en": "English", "zh": "中文", "ru": "Русский", "ja": "日本語"}

        site_names = {
            "en": "ClickHouse %s Documentation",
            "zh": "ClickHouse文档 %s",
            "ru": "Документация ClickHouse %s",
            "ja": "ClickHouseドキュメント %s",
        }

        assert len(site_names) == len(languages)

        site_dir = os.path.join(args.docs_output_dir, lang)

        plugins = ["macros"]
        if args.htmlproofer:
            plugins.append("htmlproofer")

        website_url = "https://clickhouse.com"
        site_name = site_names.get(lang, site_names["en"]) % ""
        site_name = site_name.replace("  ", " ")

        raw_config = dict(
            site_name=site_name,
            site_url=f"{website_url}/docs/{lang}/",
            docs_dir=os.path.join(args.docs_dir, lang),
            site_dir=site_dir,
            strict=True,
            theme=theme_cfg,
            copyright="©2016–2022 ClickHouse, Inc.",
            use_directory_urls=True,
            repo_name="ClickHouse/ClickHouse",
            repo_url="https://github.com/ClickHouse/ClickHouse/",
            edit_uri=f"edit/master/docs/{lang}",
            markdown_extensions=mdx_clickhouse.MARKDOWN_EXTENSIONS,
            plugins=plugins,
            extra=dict(
                now=datetime.datetime.now().isoformat(),
                single_page=False,
                rev=args.rev,
                rev_short=args.rev_short,
                rev_url=args.rev_url,
                website_url=website_url,
                events=args.events,
                languages=languages,
                includes_dir=os.path.join(os.path.dirname(__file__), "..", "_includes"),
                is_amp=False,
                is_blog=False,
            ),
        )

        # Clean to be safe if last build finished abnormally
        single_page.remove_temporary_files(lang, args)

        raw_config["nav"] = nav.build_docs_nav(lang, args)

        cfg = config.load_config(**raw_config)

        if not args.skip_multi_page:
            mkdocs.commands.build.build(cfg)

        if not args.skip_amp:
            amp.build_amp(lang, args, cfg)

        if not args.skip_single_page:
            single_page.build_single_page_version(
                lang, args, raw_config.get("nav"), cfg
            )

        mdx_clickhouse.PatchedMacrosPlugin.disabled = False

        logging.info(f"Finished building {lang} docs")

    except exceptions.ConfigurationError as e:
        raise SystemExit("\n" + str(e))


def build_docs(args):
    tasks = []
    for lang in args.lang.split(","):
        if lang:
            tasks.append(
                (
                    lang,
                    args,
                )
            )
    util.run_function_in_parallel(build_for_lang, tasks, threads=False)
    redirects.build_docs_redirects(args)


def build(args):
    if os.path.exists(args.output_dir):
        shutil.rmtree(args.output_dir)

    if not args.skip_website:
        website.build_website(args)

    if not args.skip_docs:
        generate_cmake_flags_files()

        build_docs(args)

    if not args.skip_blog:
        blog.build_blog(args)

    if not args.skip_website:
        website.process_benchmark_results(args)
        website.minify_website(args)
        redirects.build_static_redirects(args)


if __name__ == "__main__":
    os.chdir(os.path.join(os.path.dirname(__file__), ".."))

    # A root path to ClickHouse source code.
    src_dir = ".."

    website_dir = os.path.join(src_dir, "website")

    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument("--lang", default="en,ru,zh,ja")
    arg_parser.add_argument("--blog-lang", default="en")
    arg_parser.add_argument("--docs-dir", default=".")
    arg_parser.add_argument("--theme-dir", default=website_dir)
    arg_parser.add_argument("--website-dir", default=website_dir)
    arg_parser.add_argument("--src-dir", default=src_dir)
    arg_parser.add_argument("--blog-dir", default=os.path.join(website_dir, "blog"))
    arg_parser.add_argument("--output-dir", default="build")
    arg_parser.add_argument("--nav-limit", type=int, default="0")
    arg_parser.add_argument("--skip-multi-page", action="store_true")
    arg_parser.add_argument("--skip-single-page", action="store_true")
    arg_parser.add_argument("--skip-amp", action="store_true")
    arg_parser.add_argument("--skip-website", action="store_true")
    arg_parser.add_argument("--skip-blog", action="store_true")
    arg_parser.add_argument("--skip-git-log", action="store_true")
    arg_parser.add_argument("--skip-docs", action="store_true")
    arg_parser.add_argument("--test-only", action="store_true")
    arg_parser.add_argument("--minify", action="store_true")
    arg_parser.add_argument("--htmlproofer", action="store_true")
    arg_parser.add_argument("--no-docs-macros", action="store_true")
    arg_parser.add_argument("--save-raw-single-page", type=str)
    arg_parser.add_argument("--livereload", type=int, default="0")
    arg_parser.add_argument("--verbose", action="store_true")

    args = arg_parser.parse_args()
    args.minify = False  # TODO remove

    logging.basicConfig(
        level=logging.DEBUG if args.verbose else logging.INFO, stream=sys.stderr
    )

    logging.getLogger("MARKDOWN").setLevel(logging.INFO)

    args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), "docs")
    args.blog_output_dir = os.path.join(os.path.abspath(args.output_dir), "blog")

    from github import get_events

    args.rev = (
        subprocess.check_output("git rev-parse HEAD", shell=True)
        .decode("utf-8")
        .strip()
    )
    args.rev_short = (
        subprocess.check_output("git rev-parse --short HEAD", shell=True)
        .decode("utf-8")
        .strip()
    )
    args.rev_url = f"https://github.com/ClickHouse/ClickHouse/commit/{args.rev}"
    args.events = get_events(args)

    if args.test_only:
        args.skip_multi_page = True
        args.skip_blog = True
        args.skip_website = True
        args.skip_amp = True

    if args.skip_git_log or args.skip_amp:
        mdx_clickhouse.PatchedMacrosPlugin.skip_git_log = True

    from build import build

    build(args)

    if args.livereload:
        new_args = [arg for arg in sys.argv if not arg.startswith("--livereload")]
        new_args = sys.executable + " " + " ".join(new_args)

        server = livereload.Server()
        server.watch(
            args.docs_dir + "**/*", livereload.shell(new_args, cwd="tools", shell=True)
        )
        server.watch(
            args.website_dir + "**/*",
            livereload.shell(new_args, cwd="tools", shell=True),
        )
        server.serve(root=args.output_dir, host="0.0.0.0", port=args.livereload)
        sys.exit(0)