alioth/nixpkgs
3
0
Fork 0
forked from mirrors/nixpkgs
Code Releases Activity

Merge pull request #237557 from pennae/dedocbookify-nixos

Browse source 
nixos/doc: dedocbookify
This commit is contained in:
pennae 2023-06-19 14:05:03 +02:00 committed by GitHub
parent f2eabf314b 3e7649f01b
commit c8b4e5d557
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 106 additions and 724 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
.github/workflows/compare-manuals.sh vendored
View file

@ -1,21 +0,0 @@
#!/usr/bin/env nix-shell
#! nix-shell -i bash -p html-tidy
set -euo pipefail
shopt -s inherit_errexit
normalize() {
tidy \
--anchor-as-name no \
--coerce-endtags no \
--escape-scripts no \
--fix-backslash no \
--fix-style-tags no \
--fix-uri no \
--indent yes \
--wrap 0 \
< "$1" \
2> /dev/null
}
diff -U3 <(normalize "$1") <(normalize "$2")

10
.github/workflows/manual-nixos.yml vendored
View file

@ -27,13 +27,5 @@ jobs:
# This cache is for the nixpkgs repo checks and should not be trusted or used elsewhere.
name: nixpkgs-ci
signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
- name: Building NixOS manual with DocBook options
- name: Building NixOS manual
run: NIX_PATH=nixpkgs=$(pwd) nix-build --option restrict-eval true nixos/release.nix -A manual.x86_64-linux
- name: Building NixOS manual with Markdown options
run: |
export NIX_PATH=nixpkgs=$(pwd)
nix-build \
--option restrict-eval true \
--arg configuration '{ documentation.nixos.options.allowDocBook = false; }' \
nixos/release.nix \
-A manual.x86_64-linux

64
.github/workflows/manual-rendering.yml vendored
View file

@ -1,64 +0,0 @@
name: "Check NixOS Manual DocBook rendering against MD rendering"
on:
schedule:
# * is a special character in YAML so you have to quote this string
# Check every 24 hours
- cron: '0 0 * * *'
permissions:
contents: read
jobs:
check-rendering-equivalence:
permissions:
pull-requests: write # for peter-evans/create-or-update-comment to create or update comment
if: github.repository_owner == 'NixOS'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
# explicitly enable sandbox
extra_nix_config: sandbox = true
- uses: cachix/cachix-action@v12
with:
# This cache is for the nixpkgs repo checks and should not be trusted or used elsewhere.
name: nixpkgs-ci
signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
- name: Build DocBook and MD manuals
run: |
export NIX_PATH=nixpkgs=$(pwd)
nix-build \
--option restrict-eval true \
-o docbook nixos/release.nix \
-A manual.x86_64-linux
nix-build \
--option restrict-eval true \
--arg configuration '{ documentation.nixos.options.allowDocBook = false; }' \
-o md nixos/release.nix \
-A manual.x86_64-linux
- name: Compare DocBook and MD manuals
id: check
run: |
export NIX_PATH=nixpkgs=$(pwd)
.github/workflows/compare-manuals.sh \
docbook/share/doc/nixos/options.html \
md/share/doc/nixos/options.html
# if the manual can't be built we don't want to notify anyone.
# while this may temporarily hide rendering failures it will be a lot
# less noisy until all nixpkgs pull requests have stopped using
# docbook for option docs.
- name: Comment on failure
uses: peter-evans/create-or-update-comment@v3
if: ${{ failure() && steps.check.conclusion == 'failure' }}
with:
issue-number: 189318
body: |
Markdown and DocBook manuals do not agree.
Check https://github.com/NixOS/nixpkgs/actions/runs/${{ github.run_id }} for details.

2
lib/default.nix
View file

@ -138,7 +138,7 @@ let
mergeDefaultOption mergeOneOption mergeEqualOption mergeUniqueOption
getValues getFiles
optionAttrSetToDocList optionAttrSetToDocList'
scrubOptionValue literalExpression literalExample literalDocBook
scrubOptionValue literalExpression literalExample
showOption showOptionWithDefLocs showFiles
unknownModule mkOption mkPackageOption mkPackageOptionMD
mdDoc literalMD;

24
lib/modules.nix
View file

@ -134,11 +134,6 @@ let
${if prefix == []
then null # unset => visible
else "internal"} = true;
# TODO: hidden during the markdown transition to not expose downstream
# users of the docs infra to markdown if they're not ready for it.
# we don't make this visible conditionally because it can impact
# performance (https://github.com/NixOS/nixpkgs/pull/208407#issuecomment-1368246192)
visible = false;
# TODO: Change the type of this option to a submodule with a
# freeformType, so that individual arguments can be documented
# separately
@ -1146,14 +1141,11 @@ let
use = id;
};
/* Transitional version of mkAliasOptionModule that uses MD docs. */
mkAliasOptionModuleMD = from: to: doRename {
inherit from to;
visible = true;
warn = false;
use = id;
markdown = true;
};
/* Transitional version of mkAliasOptionModule that uses MD docs.
This function is no longer necessary and merely an alias of `mkAliasOptionModule`.
*/
mkAliasOptionModuleMD = mkAliasOptionModule;
/* mkDerivedConfig : Option a -> (a -> Definition b) -> Definition b
@ -1175,7 +1167,7 @@ let
(opt.highestPrio or defaultOverridePriority)
(f opt.value);
doRename = { from, to, visible, warn, use, withPriority ? true, markdown ? false }:
doRename = { from, to, visible, warn, use, withPriority ? true }:
{ config, options, ... }:
let
fromOpt = getAttrFromPath from options;
@ -1186,9 +1178,7 @@ let
{
options = setAttrByPath from (mkOption {
inherit visible;
description = if markdown
then lib.mdDoc "Alias of {option}`${showOption to}`."
else "Alias of <option>${showOption to}</option>.";
description = "Alias of {option}`${showOption to}`.";
apply = x: use (toOf config);
} // optionalAttrs (toType != null) {
type = toType;

33
lib/options.nix
View file

@ -100,10 +100,7 @@ rec {
name: mkOption {
default = false;
example = true;
description =
if name ? _type && name._type == "mdDoc"
then lib.mdDoc "Whether to enable ${name.text}."
else "Whether to enable ${name}.";
description = "Whether to enable ${name}.";
type = lib.types.bool;
};
@ -185,10 +182,10 @@ rec {
(if isList example then "pkgs." + concatStringsSep "." example else example);
});
/* Like mkPackageOption, but emit an mdDoc description instead of DocBook. */
mkPackageOptionMD = pkgs: name: extra:
let option = mkPackageOption pkgs name extra;
in option // { description = lib.mdDoc option.description; };
/* Alias of mkPackageOption. Previously used to create options with markdown
documentation, which is no longer required.
*/
mkPackageOptionMD = mkPackageOption;
/* This option accepts anything, but it does not produce any result.
@ -344,26 +341,12 @@ rec {
if ! isString text then throw "literalExpression expects a string."
else { _type = "literalExpression"; inherit text; };
literalExample = lib.warn "literalExample is deprecated, use literalExpression instead, or use literalDocBook for a non-Nix description." literalExpression;
/* For use in the `defaultText` and `example` option attributes. Causes the
given DocBook text to be inserted verbatim in the documentation, for when
a `literalExpression` would be too hard to read.
*/
literalDocBook = text:
if ! isString text then throw "literalDocBook expects a string."
else
lib.warnIf (lib.isInOldestRelease 2211)
"literalDocBook is deprecated, use literalMD instead"
{ _type = "literalDocBook"; inherit text; };
literalExample = lib.warn "literalExample is deprecated, use literalExpression instead, or use literalMD for a non-Nix description." literalExpression;
/* Transition marker for documentation that's already migrated to markdown
syntax.
syntax. This is a no-op and no longer needed.
*/
mdDoc = text:
if ! isString text then throw "mdDoc expects a string."
else { _type = "mdDoc"; inherit text; };
mdDoc = lib.id;
/* For use in the `defaultText` and `example` option attributes. Causes the
given MD text to be inserted verbatim in the documentation, for when

164
nixos/doc/manual/default.nix
View file

@ -6,7 +6,6 @@
, extraSources ? []
, baseOptionsJSON ? null
, warningsAreErrors ? true
, allowDocBook ? true
, prefix ? ../../..
}:
@ -17,10 +16,6 @@ let
lib = pkgs.lib;
docbook_xsl_ns = pkgs.docbook-xsl-ns.override {
withManOptDedupPatch = true;
};
manpageUrls = pkgs.path + "/doc/manpage-urls.json";
# We need to strip references to /nix/store/* from options,
@ -33,7 +28,7 @@ let
stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip;
optionsDoc = buildPackages.nixosOptionsDoc {
inherit options revision baseOptionsJSON warningsAreErrors allowDocBook;
inherit options revision baseOptionsJSON warningsAreErrors;
transformOptions = opt: opt // {
# Clean up declaration sites to not refer to the NixOS source tree.
declarations = map stripAnyPrefixes opt.declarations;
@ -68,73 +63,6 @@ let
optionIdPrefix = "test-opt-";
};
toc = builtins.toFile "toc.xml"
''
<toc role="chunk-toc">
<d:tocentry xmlns:d="http://docbook.org/ns/docbook" linkend="book-nixos-manual"><?dbhtml filename="index.html"?>
<d:tocentry linkend="ch-options"><?dbhtml filename="options.html"?></d:tocentry>
<d:tocentry linkend="ch-release-notes"><?dbhtml filename="release-notes.html"?></d:tocentry>
</d:tocentry>
</toc>
'';
manualXsltprocOptions = toString [
"--param chapter.autolabel 0"
"--param part.autolabel 0"
"--param preface.autolabel 0"
"--param reference.autolabel 0"
"--param section.autolabel 0"
"--stringparam html.stylesheet 'style.css overrides.css highlightjs/mono-blue.css'"
"--stringparam html.script './highlightjs/highlight.pack.js ./highlightjs/loader.js'"
"--param xref.with.number.and.title 0"
"--param toc.section.depth 0"
"--param generate.consistent.ids 1"
"--stringparam admon.style ''"
"--stringparam callout.graphics.extension .svg"
"--stringparam current.docid manual"
"--param chunk.section.depth 0"
"--param chunk.first.sections 1"
"--param use.id.as.filename 1"
"--stringparam chunk.toc ${toc}"
];
linterFunctions = ''
# outputs the context of an xmllint error output
# LEN lines around the failing line are printed
function context {
# length of context
local LEN=6
# lines to print before error line
local BEFORE=4
# xmllint output lines are:
# file.xml:1234: there was an error on line 1234
while IFS=':' read -r file line rest; do
echo
if [[ -n "$rest" ]]; then
echo "$file:$line:$rest"
local FROM=$(($line>$BEFORE ? $line - $BEFORE : 1))
# number lines & filter context
nl --body-numbering=a "$file" | sed -n "$FROM,+$LEN p"
else
if [[ -n "$line" ]]; then
echo "$file:$line"
else
echo "$file"
fi
fi
done
}
function lintrng {
xmllint --debug --noout --nonet \
--relaxng ${docbook5}/xml/rng/docbook/docbook.rng \
"$1" \
2>&1 | context 1>&2
# ^ redirect assumes xmllint doesn’t print to stdout
}
'';
prepareManualFromMD = ''
cp -r --no-preserve=all $inputs/* .
@ -154,61 +82,13 @@ let
${testOptionsDoc.optionsJSON}/share/doc/nixos/options.json
'';
manual-combined = runCommand "nixos-manual-combined"
{ inputs = lib.sourceFilesBySuffices ./. [ ".xml" ".md" ];
nativeBuildInputs = [ pkgs.nixos-render-docs pkgs.libxml2.bin pkgs.libxslt.bin ];
meta.description = "The NixOS manual as plain docbook XML";
}
''
${prepareManualFromMD}
nixos-render-docs -j $NIX_BUILD_CORES manual docbook \
--manpage-urls ${manpageUrls} \
--revision ${lib.escapeShellArg revision} \
./manual.md \
./manual-combined-pre.xml
xsltproc \
-o manual-combined.xml ${./../../lib/make-options-doc/postprocess-option-descriptions.xsl} \
manual-combined-pre.xml
${linterFunctions}
mkdir $out
cp manual-combined.xml $out/
lintrng $out/manual-combined.xml
'';
manpages-combined = runCommand "nixos-manpages-combined.xml"
{ nativeBuildInputs = [ buildPackages.libxml2.bin buildPackages.libxslt.bin ];
meta.description = "The NixOS manpages as plain docbook XML";
}
''
mkdir generated
cp -prd ${./man-pages.xml} man-pages.xml
ln -s ${optionsDoc.optionsDocBook} generated/options-db.xml
xmllint --xinclude --noxincludenode --output $out ./man-pages.xml
${linterFunctions}
lintrng $out
'';
in rec {
inherit (optionsDoc) optionsJSON optionsNix optionsDocBook optionsUsedDocbook;
inherit (optionsDoc) optionsJSON optionsNix optionsDocBook;
# Generate the NixOS manual.
manualHTML = runCommand "nixos-manual-html"
{ nativeBuildInputs =
if allowDocBook then [
buildPackages.libxml2.bin
buildPackages.libxslt.bin
] else [
buildPackages.nixos-render-docs
];
inputs = lib.optionals (! allowDocBook) (lib.sourceFilesBySuffices ./. [ ".md" ]);
{ nativeBuildInputs = [ buildPackages.nixos-render-docs ];
inputs = lib.sourceFilesBySuffices ./. [ ".md" ];
meta.description = "The NixOS manual in HTML format";
allowedReferences = ["out"];
}
@ -221,28 +101,12 @@ in rec {
cp ${../../../doc/overrides.css} $dst/overrides.css
cp -r ${pkgs.documentation-highlighter} $dst/highlightjs
${if allowDocBook then ''
xsltproc \
${manualXsltprocOptions} \
--stringparam id.warnings "1" \
--nonet --output $dst/ \
${docbook_xsl_ns}/xml/xsl/docbook/xhtml/chunktoc.xsl \
${manual-combined}/manual-combined.xml \
|& tee xsltproc.out
grep "^ID recommended on" xsltproc.out &>/dev/null && echo "error: some IDs are missing" && false
rm xsltproc.out
mkdir -p $dst/images/callouts
cp ${docbook_xsl_ns}/xml/xsl/docbook/images/callouts/*.svg $dst/images/callouts/
'' else ''
${prepareManualFromMD}
# TODO generator is set like this because the docbook/md manual compare workflow will
# trigger if it's different
nixos-render-docs -j $NIX_BUILD_CORES manual html \
--manpage-urls ${manpageUrls} \
--revision ${lib.escapeShellArg revision} \
--generator "DocBook XSL Stylesheets V${docbook_xsl_ns.version}" \
--generator "nixos-render-docs ${lib.version}" \
--stylesheet style.css \
--stylesheet overrides.css \
--stylesheet highlightjs/mono-blue.css \
@ -252,7 +116,6 @@ in rec {
--chunk-toc-depth 1 \
./manual.md \
$dst/index.html
''}
mkdir -p $out/nix-support
echo "nix-build out $out" >> $out/nix-support/hydra-build-products
@ -318,10 +181,6 @@ in rec {
manpages = runCommand "nixos-manpages"
{ nativeBuildInputs = [
buildPackages.installShellFiles
] ++ lib.optionals allowDocBook [
buildPackages.libxml2.bin
buildPackages.libxslt.bin
] ++ lib.optionals (! allowDocBook) [
buildPackages.nixos-render-docs
];
allowedReferences = ["out"];
@ -330,24 +189,11 @@ in rec {
# Generate manpages.
mkdir -p $out/share/man/man8
installManPage ${./manpages}/*
${if allowDocBook
then ''
xsltproc --nonet \
--maxdepth 6000 \
--param man.output.in.separate.dir 1 \
--param man.output.base.dir "'$out/share/man/'" \
--param man.endnotes.are.numbered 0 \
--param man.break.after.slash 1 \
${docbook_xsl_ns}/xml/xsl/docbook/manpages/docbook.xsl \
${manpages-combined}
''
else ''
mkdir -p $out/share/man/man5
nixos-render-docs -j $NIX_BUILD_CORES options manpage \
--revision ${lib.escapeShellArg revision} \
${optionsJSON}/share/doc/nixos/options.json \
$out/share/man/man5/configuration.nix.5
''}
'';
}

46
nixos/doc/manual/man-pages.xml
View file

@ -1,46 +0,0 @@
<reference xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/XInclude">
<title>NixOS Reference Pages</title>
<info>
<author>
<personname><firstname>Eelco</firstname><surname>Dolstra</surname></personname>
<contrib>Author</contrib>
</author>
<author>
<personname><othername>The Nixpkgs/NixOS contributors</othername></personname>
<contrib>Author</contrib>
</author>
<copyright><year>2007-2022</year><holder>Eelco Dolstra and the Nixpkgs/NixOS contributors</holder>
</copyright>
</info>
<refentry>
<refmeta>
<refentrytitle><filename>configuration.nix</filename>
</refentrytitle><manvolnum>5</manvolnum>
<refmiscinfo class="source">NixOS</refmiscinfo>
<!-- <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo> -->
</refmeta>
<refnamediv>
<refname><filename>configuration.nix</filename></refname>
<refpurpose>NixOS system configuration specification</refpurpose>
</refnamediv>
<refsection>
<title>Description</title>
<para>
The file <filename>/etc/nixos/configuration.nix</filename> contains the
declarative specification of your NixOS system configuration. The command
<command>nixos-rebuild</command> takes this file and realises the system
configuration specified therein.
</para>
</refsection>
<refsection>
<title>Options</title>
<para>
You can use the following options in <filename>configuration.nix</filename>.
</para>
<xi:include href="./generated/options-db.xml"
xpointer="configuration-variable-list" />
</refsection>
</refentry>
</reference>

2
nixos/doc/manual/release-notes/rl-2311.section.md
View file

@ -58,6 +58,8 @@
- A new option was added to the virtualisation module that enables specifying explicitly named network interfaces in QEMU VMs. The existing `virtualisation.vlans` is still supported for cases where the name of the network interface is irrelevant.
- DocBook option documentation is no longer supported, all module documentation now uses markdown.
- `services.nginx` gained a `defaultListen` option at server-level with support for PROXY protocol listeners, also `proxyProtocol` is now exposed in `services.nginx.virtualHosts.<name>.listen` option. It is now possible to run PROXY listeners and non-PROXY listeners at a server-level, see [#213510](https://github.com/NixOS/nixpkgs/pull/213510/) for more details.
## Nixpkgs internals {#sec-release-23.11-nixpkgs-internals}

47
nixos/lib/make-options-doc/default.nix
View file

@ -39,12 +39,17 @@
# allow docbook option docs if `true`. only markdown documentation is allowed when set to
# `false`, and a different renderer may be used with different bugs and performance
# characteristics but (hopefully) indistinguishable output.
, allowDocBook ? true
# deprecated since 23.11.
# TODO remove in a while.
, allowDocBook ? false
# whether lib.mdDoc is required for descriptions to be read as markdown.
# !!! when this is eventually flipped to true, `lib.doRename` should also default to emitting Markdown
, markdownByDefault ? false
# deprecated since 23.11.
# TODO remove in a while.
, markdownByDefault ? true
}:
assert markdownByDefault && ! allowDocBook;
let
rawOpts = lib.optionAttrSetToDocList options;
transformedOpts = map transformOptions rawOpts;
@ -134,10 +139,17 @@ in rec {
TOUCH_IF_DB=$dst/.used-docbook \
python ${./mergeJSON.py} \
${lib.optionalString warningsAreErrors "--warnings-are-errors"} \
${if allowDocBook then "--warn-on-docbook" else "--error-on-docbook"} \
$baseJSON $options \
> $dst/options.json
if grep /nixpkgs/nixos/modules $dst/options.json; then
echo "The manual appears to depend on the location of Nixpkgs, which is bad"
echo "since this prevents sharing via the NixOS channel. This is typically"
echo "caused by an option default that refers to a relative path (see above"
echo "for hints about the offending path)."
exit 1
fi
brotli -9 < $dst/options.json > $dst/options.json.br
mkdir -p $out/nix-support
@ -145,15 +157,8 @@ in rec {
echo "file json-br $dst/options.json.br" >> $out/nix-support/hydra-build-products
'';
optionsUsedDocbook = pkgs.runCommand "options-used-docbook" {} ''
if [ -e ${optionsJSON}/share/doc/nixos/.used-docbook ]; then
echo 1
else
echo 0
fi >"$out"
'';
optionsDocBook = pkgs.runCommand "options-docbook.xml" {
optionsDocBook = lib.warn "optionsDocBook is deprecated since 23.11 and will be removed in 24.05"
(pkgs.runCommand "options-docbook.xml" {
nativeBuildInputs = [
pkgs.nixos-render-docs
];
@ -164,19 +169,7 @@ in rec {
--document-type ${lib.escapeShellArg documentType} \
--varlist-id ${lib.escapeShellArg variablelistId} \
--id-prefix ${lib.escapeShellArg optionIdPrefix} \
${lib.optionalString markdownByDefault "--markdown-by-default"} \
${optionsJSON}/share/doc/nixos/options.json \
options.xml
if grep /nixpkgs/nixos/modules options.xml; then
echo "The manual appears to depend on the location of Nixpkgs, which is bad"
echo "since this prevents sharing via the NixOS channel. This is typically"
echo "caused by an option default that refers to a relative path (see above"
echo "for hints about the offending path)."
exit 1
fi
${pkgs.libxslt.bin}/bin/xsltproc \
-o "$out" ${./postprocess-option-descriptions.xsl} options.xml
'';
"$out"
'');
}

60
nixos/lib/make-options-doc/mergeJSON.py
View file

@ -43,19 +43,11 @@ def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]:
return result
warningsAreErrors = False
warnOnDocbook = False
errorOnDocbook = False
optOffset = 0
for arg in sys.argv[1:]:
if arg == "--warnings-are-errors":
optOffset += 1
warningsAreErrors = True
if arg == "--warn-on-docbook":
optOffset += 1
warnOnDocbook = True
elif arg == "--error-on-docbook":
optOffset += 1
errorOnDocbook = True
options = pivot(json.load(open(sys.argv[1 + optOffset], 'r')))
overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r')))
@ -84,38 +76,10 @@ for (k, v) in overrides.items():
severity = "error" if warningsAreErrors else "warning"
def is_docbook(o, key):
val = o.get(key, {})
if not isinstance(val, dict):
return False
return val.get('_type', '') == 'literalDocBook'
# check that every option has a description
hasWarnings = False
hasErrors = False
hasDocBook = False
for (k, v) in options.items():
if warnOnDocbook or errorOnDocbook:
kind = "error" if errorOnDocbook else "warning"
if isinstance(v.value.get('description', {}), str):
hasErrors |= errorOnDocbook
hasDocBook = True
print(
f"\x1b[1;31m{kind}: option {v.name} description uses DocBook\x1b[0m",
file=sys.stderr)
elif is_docbook(v.value, 'defaultText'):
hasErrors |= errorOnDocbook
hasDocBook = True
print(
f"\x1b[1;31m{kind}: option {v.name} default uses DocBook\x1b[0m",
file=sys.stderr)
elif is_docbook(v.value, 'example'):
hasErrors |= errorOnDocbook
hasDocBook = True
print(
f"\x1b[1;31m{kind}: option {v.name} example uses DocBook\x1b[0m",
file=sys.stderr)
if v.value.get('description', None) is None:
hasWarnings = True
print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr)
@ -126,30 +90,6 @@ for (k, v) in options.items():
f"\x1b[1;31m{severity}: option {v.name} has no type. Please specify a valid type, see " +
"https://nixos.org/manual/nixos/stable/index.html#sec-option-types\x1b[0m", file=sys.stderr)
if hasDocBook:
(why, what) = (
("disallowed for in-tree modules", "contribution") if errorOnDocbook
else ("deprecated for option documentation", "module")
)
print("Explanation: The documentation contains descriptions, examples, or defaults written in DocBook. " +
"NixOS is in the process of migrating from DocBook to Markdown, and " +
f"DocBook is {why}. To change your {what} to "+
"use Markdown, apply mdDoc and literalMD and use the *MD variants of option creation " +
"functions where they are available. For example:\n" +
"\n" +
" example.foo = mkOption {\n" +
" description = lib.mdDoc ''your description'';\n" +
" defaultText = lib.literalMD ''your description of default'';\n" +
" };\n" +
"\n" +
" example.enable = mkEnableOption (lib.mdDoc ''your thing'');\n" +
" example.package = mkPackageOptionMD pkgs \"your-package\" {};\n" +
" imports = [ (mkAliasOptionModuleMD [ \"example\" \"args\" ] [ \"example\" \"settings\" ]) ];",
file = sys.stderr)
with open(os.getenv('TOUCH_IF_DB'), 'x'):
# just make sure it exists
pass
if hasErrors:
sys.exit(1)
if hasWarnings and warningsAreErrors:

115
nixos/lib/make-options-doc/postprocess-option-descriptions.xsl
View file

@ -1,115 +0,0 @@
<?xml version="1.0"?>
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:str="http://exslt.org/strings"
xmlns:exsl="http://exslt.org/common"
xmlns:db="http://docbook.org/ns/docbook"
xmlns:nixos="tag:nixos.org"
extension-element-prefixes="str exsl">
<xsl:output method='xml' encoding="UTF-8" />
<xsl:template match="@*|node()">
<xsl:copy>
<xsl:apply-templates select="@*|node()" />
</xsl:copy>
</xsl:template>
<xsl:template name="break-up-description">
<xsl:param name="input" />
<xsl:param name="buffer" />
<!-- Every time we have two newlines following each other, we want to
break it into </para><para>. -->
<xsl:variable name="parbreak" select="'&#xa;&#xa;'" />
<!-- Similar to "(head:tail) = input" in Haskell. -->
<xsl:variable name="head" select="$input[1]" />
<xsl:variable name="tail" select="$input[position() &gt; 1]" />
<xsl:choose>
<xsl:when test="$head/self::text() and contains($head, $parbreak)">
<!-- If the haystack provided to str:split() directly starts or
ends with $parbreak, it doesn't generate a <token/> for that,
so we are doing this here. -->
<xsl:variable name="splitted-raw">
<xsl:if test="starts-with($head, $parbreak)"><token /></xsl:if>
<xsl:for-each select="str:split($head, $parbreak)">
<token><xsl:value-of select="node()" /></token>
</xsl:for-each>
<!-- Something like ends-with($head, $parbreak), but there is
no ends-with() in XSLT, so we need to use substring(). -->
<xsl:if test="
substring($head, string-length($head) -
string-length($parbreak) + 1) = $parbreak
"><token /></xsl:if>
</xsl:variable>
<xsl:variable name="splitted"
select="exsl:node-set($splitted-raw)/token" />
<!-- The buffer we had so far didn't contain any text nodes that
contain a $parbreak, so we can put the buffer along with the
first token of $splitted into a para element. -->
<para xmlns="http://docbook.org/ns/docbook">
<xsl:apply-templates select="exsl:node-set($buffer)" />
<xsl:apply-templates select="$splitted[1]/node()" />
</para>
<!-- We have already emitted the first splitted result, so the
last result is going to be set as the new $buffer later
because its contents may not be directly followed up by a
$parbreak. -->
<xsl:for-each select="$splitted[position() &gt; 1
and position() &lt; last()]">
<para xmlns="http://docbook.org/ns/docbook">
<xsl:apply-templates select="node()" />
</para>
</xsl:for-each>
<xsl:call-template name="break-up-description">
<xsl:with-param name="input" select="$tail" />
<xsl:with-param name="buffer" select="$splitted[last()]/node()" />
</xsl:call-template>
</xsl:when>
<!-- Either non-text node or one without $parbreak, which we just
want to buffer and continue recursing. -->
<xsl:when test="$input">
<xsl:call-template name="break-up-description">
<xsl:with-param name="input" select="$tail" />
<!-- This essentially appends $head to $buffer. -->
<xsl:with-param name="buffer">
<xsl:if test="$buffer">
<xsl:for-each select="exsl:node-set($buffer)">
<xsl:apply-templates select="." />
</xsl:for-each>
</xsl:if>
<xsl:apply-templates select="$head" />
</xsl:with-param>
</xsl:call-template>
</xsl:when>
<!-- No more $input, just put the remaining $buffer in a para. -->
<xsl:otherwise>
<para xmlns="http://docbook.org/ns/docbook">
<xsl:apply-templates select="exsl:node-set($buffer)" />
</para>
</xsl:otherwise>
</xsl:choose>
</xsl:template>
<xsl:template match="nixos:option-description">
<xsl:choose>
<!--
Only process nodes that are comprised of a single <para/> element,
because if that's not the case the description already contains
</para><para> in between and we need no further processing.
-->
<xsl:when test="count(db:para) > 1">
<xsl:apply-templates select="node()" />
</xsl:when>
<xsl:otherwise>
<xsl:call-template name="break-up-description">
<xsl:with-param name="input"
select="exsl:node-set(db:para/node())" />
</xsl:call-template>
</xsl:otherwise>
</xsl:choose>
</xsl:template>
</xsl:stylesheet>

30
nixos/modules/misc/documentation.nix
View file

@ -107,7 +107,7 @@ let
} >&2
'';
inherit (cfg.nixos.options) warningsAreErrors allowDocBook;
inherit (cfg.nixos.options) warningsAreErrors;
};
@ -160,6 +160,9 @@ in
(mkRenamedOptionModule [ "programs" "info" "enable" ] [ "documentation" "info" "enable" ])
(mkRenamedOptionModule [ "programs" "man" "enable" ] [ "documentation" "man" "enable" ])
(mkRenamedOptionModule [ "services" "nixosManual" "enable" ] [ "documentation" "nixos" "enable" ])
(mkRemovedOptionModule
[ "documentation" "nixos" "options" "allowDocBook" ]
"DocBook option documentation is no longer supported")
];
options = {
@ -264,23 +267,6 @@ in
'';
};
nixos.options.allowDocBook = mkOption {
type = types.bool;
default = true;
description = lib.mdDoc ''
Whether to allow DocBook option docs. When set to `false` all option using
DocBook documentation will cause a manual build error; additionally a new
renderer may be used.
::: {.note}
The `false` setting for this option is not yet fully supported. While it
should work fine and produce the same output as the previous toolchain
using DocBook it may not work in all circumstances. Whether markdown option
documentation is allowed is independent of this option.
:::
'';
};
nixos.options.warningsAreErrors = mkOption {
type = types.bool;
default = true;
@ -359,14 +345,6 @@ in
(mkIf cfg.nixos.enable {
system.build.manual = manual;
system.activationScripts.check-manual-docbook = ''
if [[ $(cat ${manual.optionsUsedDocbook}) = 1 ]]; then
echo -e "\e[31;1mwarning\e[0m: This configuration contains option documentation in docbook." \
"Support for docbook is deprecated and will be removed after NixOS 23.05." \
"See nix-store --read-log ${builtins.unsafeDiscardStringContext manual.optionsJSON.drvPath}"
fi
'';
environment.systemPackages = []
++ optional cfg.man.enable manual.manpages
++ optionals cfg.doc.enable [ manual.manualHTML nixos-help ];

6
pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/default.nix
View file

@ -1,4 +1,4 @@
{ lib, stdenv, substituteAll, fetchurl, fetchpatch, findXMLCatalogs, writeScriptBin, ruby, bash, withManOptDedupPatch ? false }:
{ lib, stdenv, substituteAll, fetchurl, fetchpatch, findXMLCatalogs, writeScriptBin, ruby, bash }:
let
@ -36,10 +36,6 @@ let
src = ./catalog-legacy-uris.patch;
inherit legacySuffix suffix version;
})
] ++ lib.optionals withManOptDedupPatch [
# Fixes https://github.com/NixOS/nixpkgs/issues/166304
# https://github.com/docbook/xslt10-stylesheets/pull/241
./fix-man-options-duplication.patch
];
propagatedBuildInputs = [ findXMLCatalogs ];

11
pkgs/data/sgml+xml/stylesheets/xslt/docbook-xsl/fix-man-options-duplication.patch
View file

@ -1,11 +0,0 @@
--- a/manpages/lists.xsl
+++ b/manpages/lists.xsl
@@ -110,7 +110,7 @@
<xsl:text>.RE&#10;</xsl:text>
</xsl:template>
-<xsl:template match="d:varlistentry/d:term"/>
+<xsl:template match="d:varlistentry/d:term" priority="1"/>
<xsl:template match="d:glossentry/d:glossterm"/>
<xsl:template match="d:variablelist[ancestor::d:listitem or ancestor::d:step or ancestor::d:glossdef]|

4
pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py
View file

@ -207,7 +207,7 @@ class ManualDocBookRenderer(RendererMixin, DocBookRenderer):
raise RuntimeError(f"rendering {path}") from e
return "".join(result)
def included_options(self, token: Token, tokens: Sequence[Token], i: int) -> str:
conv = options.DocBookConverter(self._manpage_urls, self._revision, False, 'fragment',
conv = options.DocBookConverter(self._manpage_urls, self._revision, 'fragment',
token.meta['list-id'], token.meta['id-prefix'])
conv.add_options(token.meta['source'])
return conv.finalize(fragment=True)
@ -469,7 +469,7 @@ class ManualHTMLRenderer(RendererMixin, HTMLRenderer):
return "".join(outer)
def included_options(self, token: Token, tokens: Sequence[Token], i: int) -> str:
conv = options.HTMLConverter(self._manpage_urls, self._revision, False,
conv = options.HTMLConverter(self._manpage_urls, self._revision,
token.meta['list-id'], token.meta['id-prefix'],
self._xref_targets)
conv.add_options(token.meta['source'])

121
pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py
View file

@ -37,11 +37,10 @@ class BaseConverter(Converter[md.TR], Generic[md.TR]):
_options: dict[str, RenderedOption]
def __init__(self, revision: str, markdown_by_default: bool):
def __init__(self, revision: str):
super().__init__()
self._options = {}
self._revision = revision
self._markdown_by_default = markdown_by_default
def _sorted_options(self) -> list[tuple[str, RenderedOption]]:
keys = list(self._options.keys())
@ -106,7 +105,7 @@ class BaseConverter(Converter[md.TR], Generic[md.TR]):
return []
def _render_description(self, desc: str | dict[str, str]) -> list[str]:
if isinstance(desc, str) and self._markdown_by_default:
if isinstance(desc, str):
return [ self._render(desc) ] if desc else []
elif isinstance(desc, dict) and desc.get('_type') == 'mdDoc':
return [ self._render(desc['text']) ] if desc['text'] else []
@ -198,35 +197,22 @@ class DocBookConverter(BaseConverter[OptionsDocBookRenderer]):
def __init__(self, manpage_urls: Mapping[str, str],
revision: str,
markdown_by_default: bool,
document_type: str,
varlist_id: str,
id_prefix: str):
super().__init__(revision, markdown_by_default)
super().__init__(revision)
self._renderer = OptionsDocBookRenderer(manpage_urls)
self._document_type = document_type
self._varlist_id = varlist_id
self._id_prefix = id_prefix
def _parallel_render_prepare(self) -> Any:
return (self._renderer._manpage_urls, self._revision, self._markdown_by_default, self._document_type,
return (self._renderer._manpage_urls, self._revision, self._document_type,
self._varlist_id, self._id_prefix)
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> DocBookConverter:
return cls(*a)
def _render_code(self, option: dict[str, Any], key: str) -> list[str]:
if lit := option_is(option, key, 'literalDocBook'):
return [ f"<para><emphasis>{key.capitalize()}:</emphasis> {lit['text']}</para>" ]
else:
return super()._render_code(option, key)
def _render_description(self, desc: str | dict[str, Any]) -> list[str]:
if isinstance(desc, str) and not self._markdown_by_default:
return [ f"<nixos:option-description><para>{desc}</para></nixos:option-description>" ]
else:
return super()._render_description(desc)
def _related_packages_header(self) -> list[str]:
return [
"<para>",
@ -300,19 +286,19 @@ class ManpageConverter(BaseConverter[OptionsManpageRenderer]):
_options_by_id: dict[str, str]
_links_in_last_description: Optional[list[str]] = None
def __init__(self, revision: str, markdown_by_default: bool,
def __init__(self, revision: str,
*,
# only for parallel rendering
_options_by_id: Optional[dict[str, str]] = None):
super().__init__(revision, markdown_by_default)
super().__init__(revision)
self._options_by_id = _options_by_id or {}
self._renderer = OptionsManpageRenderer({}, self._options_by_id)
def _parallel_render_prepare(self) -> Any:
return ((self._revision, self._markdown_by_default), { '_options_by_id': self._options_by_id })
return (self._revision, { '_options_by_id': self._options_by_id })
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> ManpageConverter:
return cls(*a[0], **a[1])
return cls(a[0], **a[1])
def _render_option(self, name: str, option: dict[str, Any]) -> RenderedOption:
links = self._renderer.link_footnotes = []
@ -326,21 +312,12 @@ class ManpageConverter(BaseConverter[OptionsManpageRenderer]):
return super().add_options(options)
def _render_code(self, option: dict[str, Any], key: str) -> list[str]:
if lit := option_is(option, key, 'literalDocBook'):
raise RuntimeError("can't render manpages in the presence of docbook")
else:
try:
self._renderer.inline_code_is_quoted = False
return super()._render_code(option, key)
finally:
self._renderer.inline_code_is_quoted = True
def _render_description(self, desc: str | dict[str, Any]) -> list[str]:
if isinstance(desc, str) and not self._markdown_by_default:
raise RuntimeError("can't render manpages in the presence of docbook")
else:
return super()._render_description(desc)
def _related_packages_header(self) -> list[str]:
return [
'\\fIRelated packages:\\fP',
@ -420,32 +397,16 @@ class OptionsCommonMarkRenderer(OptionDocsRestrictions, CommonMarkRenderer):
class CommonMarkConverter(BaseConverter[OptionsCommonMarkRenderer]):
__option_block_separator__ = ""
def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool):
super().__init__(revision, markdown_by_default)
def __init__(self, manpage_urls: Mapping[str, str], revision: str):
super().__init__(revision)
self._renderer = OptionsCommonMarkRenderer(manpage_urls)
def _parallel_render_prepare(self) -> Any:
return (self._renderer._manpage_urls, self._revision, self._markdown_by_default)
return (self._renderer._manpage_urls, self._revision)
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> CommonMarkConverter:
return cls(*a)
def _render_code(self, option: dict[str, Any], key: str) -> list[str]:
# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
# incorrect, since users rely on it.
if lit := option_is(option, key, 'literalDocBook'):
return [ f"*{key.capitalize()}:* {lit['text']}" ]
else:
return super()._render_code(option, key)
def _render_description(self, desc: str | dict[str, Any]) -> list[str]:
# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
# incorrect, since users rely on it.
if isinstance(desc, str) and not self._markdown_by_default:
return [ desc ]
else:
return super()._render_description(desc)
def _related_packages_header(self) -> list[str]:
return [ "*Related packages:*" ]
@ -476,32 +437,16 @@ class OptionsAsciiDocRenderer(OptionDocsRestrictions, AsciiDocRenderer):
class AsciiDocConverter(BaseConverter[OptionsAsciiDocRenderer]):
__option_block_separator__ = ""
def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool):
super().__init__(revision, markdown_by_default)
def __init__(self, manpage_urls: Mapping[str, str], revision: str):
super().__init__(revision)
self._renderer = OptionsAsciiDocRenderer(manpage_urls)
def _parallel_render_prepare(self) -> Any:
return (self._renderer._manpage_urls, self._revision, self._markdown_by_default)
return (self._renderer._manpage_urls, self._revision)
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter:
return cls(*a)
def _render_code(self, option: dict[str, Any], key: str) -> list[str]:
# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
# incorrect, since users rely on it.
if lit := option_is(option, key, 'literalDocBook'):
return [ f"*{key.capitalize()}:* {lit['text']}" ]
else:
return super()._render_code(option, key)
def _render_description(self, desc: str | dict[str, Any]) -> list[str]:
# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
# incorrect, since users rely on it.
if isinstance(desc, str) and not self._markdown_by_default:
return [ desc ]
else:
return super()._render_description(desc)
def _related_packages_header(self) -> list[str]:
return [ "__Related packages:__" ]
@ -541,33 +486,21 @@ class OptionsHTMLRenderer(OptionDocsRestrictions, HTMLRenderer):
class HTMLConverter(BaseConverter[OptionsHTMLRenderer]):
__option_block_separator__ = ""
def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool,
def __init__(self, manpage_urls: Mapping[str, str], revision: str,
varlist_id: str, id_prefix: str, xref_targets: Mapping[str, XrefTarget]):
super().__init__(revision, markdown_by_default)
super().__init__(revision)
self._xref_targets = xref_targets
self._varlist_id = varlist_id
self._id_prefix = id_prefix
self._renderer = OptionsHTMLRenderer(manpage_urls, self._xref_targets)
def _parallel_render_prepare(self) -> Any:
return (self._renderer._manpage_urls, self._revision, self._markdown_by_default,
return (self._renderer._manpage_urls, self._revision,
self._varlist_id, self._id_prefix, self._xref_targets)
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> HTMLConverter:
return cls(*a)
def _render_code(self, option: dict[str, Any], key: str) -> list[str]:
if lit := option_is(option, key, 'literalDocBook'):
raise RuntimeError("can't render html in the presence of docbook")
else:
return super()._render_code(option, key)
def _render_description(self, desc: str | dict[str, Any]) -> list[str]:
if isinstance(desc, str) and not self._markdown_by_default:
raise RuntimeError("can't render html in the presence of docbook")
else:
return super()._render_description(desc)
def _related_packages_header(self) -> list[str]:
return [
'<p><span class="emphasis"><em>Related packages:</em></span></p>',
@ -635,7 +568,6 @@ def _build_cli_db(p: argparse.ArgumentParser) -> None:
p.add_argument('--document-type', required=True)
p.add_argument('--varlist-id', required=True)
p.add_argument('--id-prefix', required=True)
p.add_argument('--markdown-by-default', default=False, action='store_true')
p.add_argument("infile")
p.add_argument("outfile")
@ -647,14 +579,12 @@ def _build_cli_manpage(p: argparse.ArgumentParser) -> None:
def _build_cli_commonmark(p: argparse.ArgumentParser) -> None:
p.add_argument('--manpage-urls', required=True)
p.add_argument('--revision', required=True)
p.add_argument('--markdown-by-default', default=False, action='store_true')
p.add_argument("infile")
p.add_argument("outfile")
def _build_cli_asciidoc(p: argparse.ArgumentParser) -> None:
p.add_argument('--manpage-urls', required=True)
p.add_argument('--revision', required=True)
p.add_argument('--markdown-by-default', default=False, action='store_true')
p.add_argument("infile")
p.add_argument("outfile")
@ -663,7 +593,6 @@ def _run_cli_db(args: argparse.Namespace) -> None:
md = DocBookConverter(
json.load(manpage_urls),
revision = args.revision,
markdown_by_default = args.markdown_by_default,
document_type = args.document_type,
varlist_id = args.varlist_id,
id_prefix = args.id_prefix)
@ -674,11 +603,7 @@ def _run_cli_db(args: argparse.Namespace) -> None:
f.write(md.finalize())
def _run_cli_manpage(args: argparse.Namespace) -> None:
md = ManpageConverter(
revision = args.revision,
# manpage rendering only works if there's no docbook, so we can
# also set markdown_by_default with no ill effects.
markdown_by_default = True)
md = ManpageConverter(revision = args.revision)
with open(args.infile, 'r') as f:
md.add_options(json.load(f))
@ -687,10 +612,7 @@ def _run_cli_manpage(args: argparse.Namespace) -> None:
def _run_cli_commonmark(args: argparse.Namespace) -> None:
with open(args.manpage_urls, 'r') as manpage_urls:
md = CommonMarkConverter(
json.load(manpage_urls),
revision = args.revision,
markdown_by_default = args.markdown_by_default)
md = CommonMarkConverter(json.load(manpage_urls), revision = args.revision)
with open(args.infile, 'r') as f:
md.add_options(json.load(f))
@ -699,10 +621,7 @@ def _run_cli_commonmark(args: argparse.Namespace) -> None:
def _run_cli_asciidoc(args: argparse.Namespace) -> None:
with open(args.manpage_urls, 'r') as manpage_urls:
md = AsciiDocConverter(
json.load(manpage_urls),
revision = args.revision,
markdown_by_default = args.markdown_by_default)
md = AsciiDocConverter(json.load(manpage_urls), revision = args.revision)
with open(args.infile, 'r') as f:
md.add_options(json.load(f))

2
pkgs/tools/nix/nixos-render-docs/src/tests/test_options.py
View file

@ -4,7 +4,7 @@ from markdown_it.token import Token
import pytest
def test_option_headings() -> None:
c = nixos_render_docs.options.DocBookConverter({}, 'local', False, 'none', 'vars', 'opt-')
c = nixos_render_docs.options.DocBookConverter({}, 'local', 'none', 'vars', 'opt-')
with pytest.raises(RuntimeError) as exc:
c._render("# foo")
assert exc.value.args[0] == 'md token not supported in options doc'