Initial version of documentation generator

documentation-tools/filter.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+"""
+Filter program for transforming the Pandoc AST
+"""
+
+import json
+import re
+import sys
+
+from build import SHIFT_HEADING
+
+
+def text_content(content):
+    text = ""
+    for block in content:
+        if block["t"] == "Space":
+            text += " "
+        elif block["t"] == "Str":
+            text += block["c"]
+    return text
+
+
+def convert_framgent(*titles):
+    titles = list(titles)
+    for i, title in enumerate(titles):
+        title = re.sub(r"\s", "-", title)
+        title = re.sub(r"[^\w-]", "", title)
+        titles[i] = title.lower()
+    return "/".join(titles)
+
+
+def main():
+    data = json.load(sys.stdin)
+
+    # Use hierachical link fragments (e.g. #heading/subheading)
+    headings = []
+    for block in data["blocks"]:
+        if block["t"] != "Header":
+            continue
+        level, (attr_id, attr_class, attr_name), content = block["c"]
+        shifted_level = level - SHIFT_HEADING
+        title = text_content(content)
+        headings = headings[:shifted_level - 1]
+        while len(headings) < shifted_level - 1:
+            headings.append("")
+        headings.append(title)
+        full_attr_id = convert_framgent(*headings)
+        block["c"] = [level, [full_attr_id, attr_class, attr_name], content]
+
+    json.dump(data, sys.stdout)
+
+
+if __name__ == "__main__":
+    main()

documentation-tools/generate.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+
+"""
+Documentation generator
+
+Generates the documentation for every Git branch and commits it.
+Gracefully handles conflicting commits.
+"""
+
+import contextlib
+import glob
+import os
+import re
+import shutil
+import subprocess
+import sys
+from tempfile import TemporaryDirectory
+
+REMOTE = "origin"
+GIT_CONFIG = {"protocol.version": "2",
+              "user.email": "<>",
+              "user.name": "Github Actions"}
+COMMIT_MESSAGE = "Generate documentation"
+DOCUMENTATION_SRC = "DOCUMENTATION.md"
+TARGET_DIR = "beta"
+SHIFT_HEADING = 1
+TOOLS_PATH = os.path.dirname(__file__)
+TEMPLATE_PATH = os.path.join(TOOLS_PATH, "template.html")
+FILTER_EXE = os.path.join(TOOLS_PATH, "filter.py")
+POSTPROCESSOR_EXE = os.path.join(TOOLS_PATH, "postprocessor.py")
+PANDOC_EXE = "pandoc"
+PANDOC_DOWNLOAD = "https://github.com/jgm/pandoc/releases/download/2.9.2/pandoc-2.9.2-1-amd64.deb"
+
+
+def convert_doc(src_path, to_path, branch, branches):
+    subprocess.run([
+        PANDOC_EXE,
+        "--from=gfm",
+        "--to=html5",
+        os.path.abspath(src_path),
+        "--toc",
+        "--template=%s" % os.path.basename(TEMPLATE_PATH),
+        "--output=%s" % os.path.abspath(to_path),
+        "--section-divs",
+        "--shift-heading-level-by=%d" % SHIFT_HEADING,
+        "--toc-depth=4",
+        "--filter=%s" % os.path.abspath(FILTER_EXE),
+        "--variable=branch=%s" % branch,
+        *["--variable=branches=%s" % b for b in branches]],
+        check=True, cwd=os.path.dirname(TEMPLATE_PATH))
+    with open(to_path, "rb+") as f:
+        data = subprocess.run([POSTPROCESSOR_EXE], input=f.read(),
+                              stdout=subprocess.PIPE, check=True).stdout
+        f.seek(0)
+        f.truncate()
+        f.write(data)
+
+
+def install_dependencies():
+    subprocess.run([sys.executable, "-m", "pip", "install", "beautifulsoup4"],
+                   check=True)
+    with TemporaryDirectory() as temp:
+        subprocess.run(["curl", "--location", "--output", "pandoc.deb",
+                        PANDOC_DOWNLOAD], check=True, cwd=temp)
+        subprocess.run(["apt", "install", "--assume-yes", "./pandoc.deb"],
+                       check=True, cwd=temp)
+
+
+def natural_sort_key(s):
+    # https://stackoverflow.com/a/16090640
+    return [int(part) if part.isdigit() else part.lower()
+            for part in re.split(r"(\d+)", s)]
+
+
+def run_git(*args):
+    config_args = []
+    for key, value in GIT_CONFIG.items():
+        config_args.extend(["-c", "%s=%s" % (key, value)])
+    output = subprocess.run(["git", *config_args, *args],
+                            stdout=subprocess.PIPE, check=True,
+                            universal_newlines=True).stdout
+    return tuple(filter(None, output.split("\n")))
+
+
+def checkout(branch):
+    run_git("checkout", "--progress", "--force", "-B", branch,
+            "refs/remotes/%s/%s" % (REMOTE, branch))
+
+
+def run_git_fetch_and_restart_if_changed(remote_commits, target_branch):
+    run_git("fetch", "--no-tags", "--prune", "--progress",
+            "--no-recurse-submodules", "--depth=1", REMOTE,
+            "+refs/heads/*:refs/remotes/%s/*" % REMOTE)
+    if remote_commits != run_git("rev-parse", "--remotes=%s" % REMOTE):
+        checkout(target_branch)
+        print("Remote changed, restarting", file=sys.stderr)
+        os.execv(__file__, sys.argv)
+
+
+def main():
+    install_dependencies()
+    target_branch, = run_git("rev-parse", "--abbrev-ref", "HEAD")
+    remote_commits = run_git("rev-parse", "--remotes=%s" % REMOTE)
+    run_git_fetch_and_restart_if_changed(remote_commits, target_branch)
+    branches = [ref[len("refs/remotes/%s/" % REMOTE):] for ref in run_git(
+        "rev-parse", "--symbolic-full-name", "--remotes=%s" % REMOTE)]
+    branches.sort(key=natural_sort_key, reverse=True)
+    os.makedirs(TARGET_DIR, exist_ok=True)
+    for path in glob.iglob(os.path.join(TARGET_DIR, "*.html")):
+        run_git("rm", "--", path)
+    with TemporaryDirectory() as temp:
+        branch_docs = {}
+        for branch in branches:
+            checkout(branch)
+            if os.path.exists(DOCUMENTATION_SRC):
+                branch_docs[branch] = os.path.join(temp, "%s.md" % branch)
+                shutil.copy(DOCUMENTATION_SRC, branch_docs[branch])
+        checkout(target_branch)
+        for branch, src_path in branch_docs.items():
+            to_path = os.path.join(TARGET_DIR, "%s.html" % branch)
+            convert_doc(src_path, to_path, branch, branches)
+            run_git("add", "--", to_path)
+    with contextlib.suppress(subprocess.CalledProcessError):
+        run_git("diff", "--cached", "--quiet")
+        print("No changes", file=sys.stderr)
+        return
+    run_git("commit", "-m", COMMIT_MESSAGE)
+    try:
+        run_git("push", REMOTE, "HEAD:%s" % target_branch)
+    except subprocess.CalledProcessError:
+        run_git_fetch_and_restart_if_changed(remote_commits, target_branch)
+        raise
+
+
+if __name__ == "__main__":
+    main()

documentation-tools/postprocessor.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+"""
+Postprocessor program for the HTML output of Pandoc
+"""
+
+import sys
+
+from bs4 import BeautifulSoup
+
+
+def add_class(element, class_):
+    element["class"] = element.get("class", []) + [class_]
+
+
+def main():
+    soup = BeautifulSoup(sys.stdin.buffer, "html.parser")
+
+    # Mark the hierachical levels in the navigation
+    for section in soup.select("section"):
+        link = soup.find("a", href="#" + section["id"])
+        if link is None:
+            continue
+        add_class(link.parent, section["class"][0])
+
+    # Mark last section
+    add_class(soup.select("section")[-1], "last")
+
+    # Wrap tables in a div container (for scrolling)
+    for table in soup.select("main table"):
+        container = soup.new_tag("div")
+        add_class(container, "tableContainer")
+        table.wrap(container)
+
+    # Add a link with the fragment to every header
+    for header in soup.select("section > *:first-child"):
+        section = header.parent
+        link = soup.new_tag("a")
+        add_class(link, "headerlink")
+        link["href"] = "#" + section["id"]
+        link.string = "¶"
+        header.append(" ")
+        header.append(link)
+
+    sys.stdout.buffer.write(soup.encode(formatter="html5") + b"\n")
+
+
+if __name__ == "__main__":
+    main()

documentation-tools/template.html

@@ -0,0 +1,53 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<meta charset="utf-8">
+<meta name="generator" content="pandoc">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<style>
+    $styles.html()$
+</style>
+$if(math)$
+    $math$
+$endif$
+<link href="assets/default.css" media="all" rel="stylesheet">
+<link href="assets/screen.css" media="screen" rel="stylesheet">
+<noscript><link href="assets/screen-noscript.css" media="screen" rel="stylesheet"></noscript>
+<link href="https://github.com/Kozea/Radicale/releases.atom" type="application/atom+xml" rel="alternate" title="Radicale Releases">
+<link href="assets/icon.png" type="image/png" rel="shortcut icon">
+<title>Radicale - Free and Open-Source CalDAV and CardDAV Server</title>
+<meta name="description" content="Free and Open-Source CalDAV and CardDAV Server">
+<script src="assets/navigation.js"></script>
+<script src="assets/document-branches.js"></script>
+<script>
+    const documentBranch = "$branch$";
+    const documentBranches = ["$for(branches)$$branches$$sep$", "$endfor$"];
+</script>
+
+<header>
+    <div class="logoContainer">
+        <h1>
+            Radicale
+            <span class="documentBranch">$branch$</span>
+            <select class="documentBranch"></select>
+        </h1>
+        <p>Free and Open-Source CalDAV and CardDAV Server</p>
+    </div>
+    <div class="linkContainer">
+        <ul>
+            <li><a href="https://community.kozea.fr">Made with ❤ by Kozea Community</a></li>
+            <li><a href="https://github.com/Kozea/Radicale">Fork me on GitHub</a></li>
+        </ul>
+    </div>
+</header>
+
+<main>
+    <nav>
+        <h2>Contents</h2>
+        $table-of-contents$
+    </nav>
+    <div class="documentContainer">
+$body$
+    </div>
+</main>