From a8d4cf149cda4f82145291cc9a6819a9baa3951e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 2 Jan 2023 14:00:58 +0100 Subject: [PATCH 1/3] doc: separate manpage URLs from the Pandoc filter Move the manpage-to-URL mapping to `doc/manpage-urls.json` so that we can reuse that file elsewhere, and generate the `link-manpages.lua` filter from that file. Also modify the Pandoc filter so that it doesn't wrap manpages that are already inside a link. Keeping a Lua filter is essential for speed: a Python filter would increase the runtime `md-to-db.sh` from ~20s to ~30s (but Python is not to blame; marshalling Pandoc types to and from JSON is a costly operation). Parsing in Lua seems tedious, so I went with the Nix way. --- doc/Makefile | 2 +- .../pandoc-filters/link-manpages.nix | 28 ++++++++++++++ .../link-unix-man-references.lua | 38 ------------------- .../contributing-to-documentation.chapter.md | 2 +- doc/default.nix | 1 + doc/manpage-urls.json | 29 ++++++++++++++ maintainers/scripts/db-to-md.sh | 2 +- nixos/doc/manual/md-to-db.sh | 4 +- 8 files changed, 64 insertions(+), 42 deletions(-) create mode 100644 doc/build-aux/pandoc-filters/link-manpages.nix delete mode 100644 doc/build-aux/pandoc-filters/link-unix-man-references.lua create mode 100644 doc/manpage-urls.json diff --git a/doc/Makefile b/doc/Makefile index f8d2d7248fab..1e54aef9bd12 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -11,7 +11,7 @@ pandoc_commonmark_enabled_extensions = +attributes+fenced_divs+footnotes+bracket pandoc_flags = --extract-media=$(pandoc_media_dir) \ --lua-filter=$(PANDOC_LUA_FILTERS_DIR)/diagram-generator.lua \ --lua-filter=build-aux/pandoc-filters/myst-reader/roles.lua \ - --lua-filter=build-aux/pandoc-filters/link-unix-man-references.lua \ + --lua-filter=$(PANDOC_LINK_MANPAGES_FILTER) \ --lua-filter=build-aux/pandoc-filters/docbook-writer/rst-roles.lua \ --lua-filter=build-aux/pandoc-filters/docbook-writer/labelless-link-is-xref.lua \ -f commonmark$(pandoc_commonmark_enabled_extensions)+smart diff --git a/doc/build-aux/pandoc-filters/link-manpages.nix b/doc/build-aux/pandoc-filters/link-manpages.nix new file mode 100644 index 000000000000..ee27c5d60812 --- /dev/null +++ b/doc/build-aux/pandoc-filters/link-manpages.nix @@ -0,0 +1,28 @@ +{ pkgs ? import ../../.. {} }: +let + inherit (pkgs) lib; + manpageURLs = builtins.fromJSON (builtins.readFile (pkgs.path + "/doc/manpage-urls.json")); +in pkgs.writeText "link-manpages.lua" '' + --[[ + Adds links to known man pages that aren't already in a link. + ]] + + local manpage_urls = { + ${lib.concatStringsSep "\n" (lib.mapAttrsToList (man: url: + " [${builtins.toJSON man}] = ${builtins.toJSON url},") manpageURLs)} + } + + traverse = 'topdown' + + -- Returning false as the second value aborts processing of child elements. + function Link(elem) + return elem, false + end + + function Code(elem) + local is_man_role = elem.classes:includes('interpreted-text') and elem.attributes['role'] == 'manpage' + if is_man_role and manpage_urls[elem.text] ~= nil then + return pandoc.Link(elem, manpage_urls[elem.text]), false + end + end +'' diff --git a/doc/build-aux/pandoc-filters/link-unix-man-references.lua b/doc/build-aux/pandoc-filters/link-unix-man-references.lua deleted file mode 100644 index b81d4a0979e7..000000000000 --- a/doc/build-aux/pandoc-filters/link-unix-man-references.lua +++ /dev/null @@ -1,38 +0,0 @@ ---[[ -Turns a manpage reference into a link, when a mapping is defined below. -]] - -local man_urls = { - ["nix.conf(5)"] = "https://nixos.org/manual/nix/stable/#sec-conf-file", - - ["journald.conf(5)"] = "https://www.freedesktop.org/software/systemd/man/journald.conf.html", - ["logind.conf(5)"] = "https://www.freedesktop.org/software/systemd/man/logind.conf.html", - ["networkd.conf(5)"] = "https://www.freedesktop.org/software/systemd/man/networkd.conf.html", - ["systemd.automount(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.automount.html", - ["systemd.exec(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.exec.html", - ["systemd.link(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.link.html", - ["systemd.mount(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.mount.html", - ["systemd.netdev(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.netdev.html", - ["systemd.network(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.network.html", - ["systemd.nspawn(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.nspawn.html", - ["systemd.path(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.path.html", - ["systemd.resource-control(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html", - ["systemd.scope(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.scope.html", - ["systemd.service(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.service.html", - ["systemd.slice(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.slice.html", - ["systemd.socket(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.socket.html", - ["systemd.timer(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.timer.html", - ["systemd.unit(5)"] = "https://www.freedesktop.org/software/systemd/man/systemd.unit.html", - ["timesyncd.conf(5)"] = "https://www.freedesktop.org/software/systemd/man/timesyncd.conf.html", - ["tmpfiles.d(5)"] = "https://www.freedesktop.org/software/systemd/man/tmpfiles.d.html", - ["systemd.time(7)"] = "https://www.freedesktop.org/software/systemd/man/systemd.time.html", - ["systemd-fstab-generator(8)"] = "https://www.freedesktop.org/software/systemd/man/systemd-fstab-generator.html", - ["systemd-networkd-wait-online.service(8)"] = "https://www.freedesktop.org/software/systemd/man/systemd-networkd-wait-online.service.html", -} - -function Code(elem) - local is_man_role = elem.classes:includes('interpreted-text') and elem.attributes['role'] == 'manpage' - if is_man_role and man_urls[elem.text] ~= nil then - return pandoc.Link(elem, man_urls[elem.text]) - end -end diff --git a/doc/contributing/contributing-to-documentation.chapter.md b/doc/contributing/contributing-to-documentation.chapter.md index 81482523cd0e..889b4114acca 100644 --- a/doc/contributing/contributing-to-documentation.chapter.md +++ b/doc/contributing/contributing-to-documentation.chapter.md @@ -53,7 +53,7 @@ Additional syntax extensions are available, though not all extensions can be use This syntax is taken from [MyST](https://myst-parser.readthedocs.io/en/latest/using/syntax.html#targets-and-cross-referencing). - []{#ssec-contributing-markup-inline-roles} - If you want to link to a man page, you can use `` {manpage}`nix.conf(5)` ``, which will turn into {manpage}`nix.conf(5)`. The references will turn into links when a mapping exists in {file}`doc/build-aux/pandoc-filters/link-unix-man-references.lua`. + If you want to link to a man page, you can use `` {manpage}`nix.conf(5)` ``, which will turn into {manpage}`nix.conf(5)`. The references will turn into links when a mapping exists in {file}`doc/manpage-urls.json`. A few markups for other kinds of literals are also available: diff --git a/doc/default.nix b/doc/default.nix index ac382ec8519c..1d5fa4811a36 100644 --- a/doc/default.nix +++ b/doc/default.nix @@ -36,4 +36,5 @@ in pkgs.stdenv.mkDerivation { # Environment variables PANDOC_LUA_FILTERS_DIR = "${pkgs.pandoc-lua-filters}/share/pandoc/filters"; + PANDOC_LINK_MANPAGES_FILTER = import build-aux/pandoc-filters/link-manpages.nix { inherit pkgs; }; } diff --git a/doc/manpage-urls.json b/doc/manpage-urls.json new file mode 100644 index 000000000000..0ff4b762bec7 --- /dev/null +++ b/doc/manpage-urls.json @@ -0,0 +1,29 @@ +{ + "nix.conf(5)": "https://nixos.org/manual/nix/stable/#sec-conf-file", + + "journald.conf(5)": "https://www.freedesktop.org/software/systemd/man/journald.conf.html", + "logind.conf(5)": "https://www.freedesktop.org/software/systemd/man/logind.conf.html", + "networkd.conf(5)": "https://www.freedesktop.org/software/systemd/man/networkd.conf.html", + "systemd.automount(5)": "https://www.freedesktop.org/software/systemd/man/systemd.automount.html", + "systemd.exec(5)": "https://www.freedesktop.org/software/systemd/man/systemd.exec.html", + "systemd.link(5)": "https://www.freedesktop.org/software/systemd/man/systemd.link.html", + "systemd.mount(5)": "https://www.freedesktop.org/software/systemd/man/systemd.mount.html", + "systemd.netdev(5)": "https://www.freedesktop.org/software/systemd/man/systemd.netdev.html", + "systemd.network(5)": "https://www.freedesktop.org/software/systemd/man/systemd.network.html", + "systemd.nspawn(5)": "https://www.freedesktop.org/software/systemd/man/systemd.nspawn.html", + "systemd.path(5)": "https://www.freedesktop.org/software/systemd/man/systemd.path.html", + "systemd.resource-control(5)": "https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html", + "systemd.scope(5)": "https://www.freedesktop.org/software/systemd/man/systemd.scope.html", + "systemd.service(5)": "https://www.freedesktop.org/software/systemd/man/systemd.service.html", + "systemd.slice(5)": "https://www.freedesktop.org/software/systemd/man/systemd.slice.html", + "systemd.socket(5)": "https://www.freedesktop.org/software/systemd/man/systemd.socket.html", + "systemd.timer(5)": "https://www.freedesktop.org/software/systemd/man/systemd.timer.html", + "systemd.unit(5)": "https://www.freedesktop.org/software/systemd/man/systemd.unit.html", + "systemd-system.conf(5)": "https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html", + "systemd-user.conf(5)": "https://www.freedesktop.org/software/systemd/man/systemd-user.conf.html", + "timesyncd.conf(5)": "https://www.freedesktop.org/software/systemd/man/timesyncd.conf.html", + "tmpfiles.d(5)": "https://www.freedesktop.org/software/systemd/man/tmpfiles.d.html", + "systemd.time(7)": "https://www.freedesktop.org/software/systemd/man/systemd.time.html", + "systemd-fstab-generator(8)": "https://www.freedesktop.org/software/systemd/man/systemd-fstab-generator.html", + "systemd-networkd-wait-online.service(8)": "https://www.freedesktop.org/software/systemd/man/systemd-networkd-wait-online.service.html" +} diff --git a/maintainers/scripts/db-to-md.sh b/maintainers/scripts/db-to-md.sh index 01357d1e2412..aa2a2775b6de 100755 --- a/maintainers/scripts/db-to-md.sh +++ b/maintainers/scripts/db-to-md.sh @@ -41,7 +41,7 @@ pandoc_flags=( # - diagram-generator.lua (we do not support that in NixOS manual to limit dependencies) # - media extraction (was only required for diagram generator) # - myst-reader/roles.lua (only relevant for MyST → DocBook) - # - link-unix-man-references.lua (links should only be added to display output) + # - link-manpages.lua (links should only be added to display output) # - docbook-writer/rst-roles.lua (only relevant for → DocBook) # - docbook-writer/labelless-link-is-xref.lua (only relevant for → DocBook) "--lua-filter=$DIR/../../doc/build-aux/pandoc-filters/docbook-reader/citerefentry-to-rst-role.lua" diff --git a/nixos/doc/manual/md-to-db.sh b/nixos/doc/manual/md-to-db.sh index 6eca9f3b2c3d..4698e94f508b 100755 --- a/nixos/doc/manual/md-to-db.sh +++ b/nixos/doc/manual/md-to-db.sh @@ -8,6 +8,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" pushd "$DIR" +link_manpages_filter=$(nix-build --no-out-link "$DIR/../../../doc/build-aux/pandoc-filters/link-manpages.nix") + # NOTE: Keep in sync with Nixpkgs manual (/doc/Makefile). # TODO: Remove raw-attribute when we can get rid of DocBook altogether. pandoc_commonmark_enabled_extensions=+attributes+fenced_divs+footnotes+bracketed_spans+definition_lists+pipe_tables+raw_attribute @@ -17,7 +19,7 @@ pandoc_flags=( # - media extraction (was only required for diagram generator) # - docbook-reader/citerefentry-to-rst-role.lua (only relevant for DocBook → MarkDown/rST/MyST) "--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/myst-reader/roles.lua" - "--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/link-unix-man-references.lua" + "--lua-filter=$link_manpages_filter" "--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/docbook-writer/rst-roles.lua" "--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/docbook-writer/html-elements.lua" "--lua-filter=$DIR/../../../doc/build-aux/pandoc-filters/docbook-writer/labelless-link-is-xref.lua" From f61e538c432a87eb0601d5da910abda131459fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 2 Jan 2023 14:09:29 +0100 Subject: [PATCH 2/3] nixos/make-options-doc: link manpages Add links to manpages without a link using the mapping defined in `doc/manpage-urls.json`, as is already done for the nixpkgs and NixOS manuals. --- nixos/lib/make-options-doc/default.nix | 2 ++ nixos/lib/make-options-doc/mergeJSON.py | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index a3436caad8f9..f2bcbba69aec 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -130,6 +130,8 @@ in rec { if baseOptionsJSON == null then builtins.toFile "base.json" "{}" else baseOptionsJSON; + + MANPAGE_URLS = pkgs.path + "/doc/manpage-urls.json"; } '' # Export list of options in different format. diff --git a/nixos/lib/make-options-doc/mergeJSON.py b/nixos/lib/make-options-doc/mergeJSON.py index 750cd24fc653..070f2d6025a2 100644 --- a/nixos/lib/make-options-doc/mergeJSON.py +++ b/nixos/lib/make-options-doc/mergeJSON.py @@ -1,5 +1,6 @@ import collections import json +import os import sys from typing import Any, Dict, List @@ -46,6 +47,8 @@ def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]: result[opt.name] = opt.value return result +manpage_urls = json.load(open(os.getenv('MANPAGE_URLS'))) + admonitions = { '.warning': 'warning', '.important': 'important', @@ -119,9 +122,14 @@ class Renderer(mistune.renderers.BaseRenderer): def env(self, text): return f"{escape(text)}" def manpage(self, page, section): + man = f"{page}({section})" title = f"{escape(page)}" vol = f"{escape(section)}" - return f"{title}{vol}" + ref = f"{title}{vol}" + if man in manpage_urls: + return self.link(manpage_urls[man], text=ref) + else: + return ref def finalize(self, data): return "".join(data) From 4fb500d62937dcce11c84e7c36fcc0d5db354d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sat, 31 Dec 2022 12:19:48 +0100 Subject: [PATCH 3/3] nixos/doc: fix some manpage references --- nixos/lib/systemd-unit-options.nix | 4 ++-- nixos/modules/services/backup/restic.nix | 2 +- nixos/modules/services/networking/kresd.nix | 6 +++--- nixos/modules/system/boot/systemd/user.nix | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nixos/lib/systemd-unit-options.nix b/nixos/lib/systemd-unit-options.nix index 9c7cb34f14b5..6c53c5e0533d 100644 --- a/nixos/lib/systemd-unit-options.nix +++ b/nixos/lib/systemd-unit-options.nix @@ -60,7 +60,7 @@ in rec { `asDropin` creates a drop-in file named `overrides.conf`. Mainly needed to define instances for systemd template units (e.g. `systemd-nspawn@mycontainer.service`). - See also systemd.unit(1). + See also {manpage}`systemd.unit(5)`. ''; }; @@ -86,7 +86,7 @@ in rec { This option creates a `.wants` symlink in the given target that exists statelessly without the need for running `systemctl enable`. - The in systemd.unit(5) manpage described `[Install]` section however is + The `[Install]` section described in {manpage}`systemd.unit(5)` however is not supported because it is a stateful process that does not fit well into the NixOS design. ''; diff --git a/nixos/modules/services/backup/restic.nix b/nixos/modules/services/backup/restic.nix index 869ed5d9976c..0d21b1e8d66a 100644 --- a/nixos/modules/services/backup/restic.nix +++ b/nixos/modules/services/backup/restic.nix @@ -132,7 +132,7 @@ in OnCalendar = "daily"; }; description = lib.mdDoc '' - When to run the backup. See man systemd.timer for details. + When to run the backup. See {manpage}`systemd.timer(5)` for details. ''; example = { OnCalendar = "00:05"; diff --git a/nixos/modules/services/networking/kresd.nix b/nixos/modules/services/networking/kresd.nix index 55af6abd5e01..3ad757133a60 100644 --- a/nixos/modules/services/networking/kresd.nix +++ b/nixos/modules/services/networking/kresd.nix @@ -79,7 +79,7 @@ in { example = [ "53" ]; description = lib.mdDoc '' What addresses and ports the server should listen on. - For detailed syntax see ListenStream in man systemd.socket. + For detailed syntax see ListenStream in {manpage}`systemd.socket(5)`. ''; }; listenTLS = mkOption { @@ -88,7 +88,7 @@ in { example = [ "198.51.100.1:853" "[2001:db8::1]:853" "853" ]; description = lib.mdDoc '' Addresses and ports on which kresd should provide DNS over TLS (see RFC 7858). - For detailed syntax see ListenStream in man systemd.socket. + For detailed syntax see ListenStream in {manpage}`systemd.socket(5)`. ''; }; listenDoH = mkOption { @@ -97,7 +97,7 @@ in { example = [ "198.51.100.1:443" "[2001:db8::1]:443" "443" ]; description = lib.mdDoc '' Addresses and ports on which kresd should provide DNS over HTTPS/2 (see RFC 8484). - For detailed syntax see ListenStream in man systemd.socket. + For detailed syntax see ListenStream in {manpage}`systemd.socket(5)`. ''; }; instances = mkOption { diff --git a/nixos/modules/system/boot/systemd/user.nix b/nixos/modules/system/boot/systemd/user.nix index 46d66fe4e688..d14d0928c53e 100644 --- a/nixos/modules/system/boot/systemd/user.nix +++ b/nixos/modules/system/boot/systemd/user.nix @@ -46,7 +46,7 @@ in { type = types.lines; example = "DefaultCPUAccounting=yes"; description = lib.mdDoc '' - Extra config options for systemd user instances. See man systemd-user.conf for + Extra config options for systemd user instances. See {manpage}`systemd-user.conf(5)` for available options. ''; };