Improve docs

.readthedocs.yml

@@ -1,11 +1,14 @@
 version: 2
 sphinx:
   configuration: docs/conf.py
-formats: all 
+formats: all
 build:
-  os: "ubuntu-20.04"
+  os: "ubuntu-24.04"
   tools:
-    python: "3.8"
+    python: "3.13"
 python:
   install:
-    - requirements: docs/requirements.txt
+    - method: uv
+      command: sync
+      groups:
+        - docs

docs/_ext/pyff_pipes.py

@@ -0,0 +1,56 @@
+"""
+Sphinx directive that documents every registered pyFF pipe from its docstring.
+
+Usage in a document::
+
+    .. pyff-pipes::
+
+Pipes are listed by the name used in pipeline YAML (the key in
+:py:data:`pyff.pipes.registry`), not by the name of the Python function.
+"""
+
+import inspect
+import re
+
+from docutils import nodes
+from docutils.statemachine import StringList
+from sphinx.util.docutils import SphinxDirective
+from sphinx.util.nodes import nested_parse_with_titles
+
+# Field lines describe the Python signature, which is irrelevant for pipeline authors.
+_PYTHON_FIELDS = re.compile(r'^\s*:(param|type|return|returns|rtype)\b')
+
+
+def _pipe_docs():
+    import pyff.builtins  # noqa: F401  (importing registers the pipes)
+    from pyff.pipes import registry
+
+    for name, func in sorted(registry.items()):
+        doc = inspect.getdoc(func)
+        # Deprecated pipes are wrapped and end up without a docstring.
+        if doc:
+            yield name, doc
+
+
+def _strip_python_fields(doc):
+    return '\n'.join(line for line in doc.splitlines() if not _PYTHON_FIELDS.match(line))
+
+
+class PyffPipes(SphinxDirective):
+    has_content = False
+
+    def run(self):
+        lines = []
+        for name, doc in _pipe_docs():
+            lines += [name, '-' * len(name), '', _strip_python_fields(doc), '']
+
+        node = nodes.section()
+        node.document = self.state.document
+        content = StringList('\n'.join(lines).splitlines(), source=f'<{self.name}>')
+        nested_parse_with_titles(self.state, content, node)
+        return node.children
+
+
+def setup(app):
+    app.add_directive('pyff-pipes', PyffPipes)
+    return {'parallel_read_safe': True, 'parallel_write_safe': True}

docs/conf.py

@@ -15,21 +15,35 @@
 
 import sphinx
 
+_here = os.path.dirname(os.path.abspath(__file__))
+
 sys.path.insert(0, os.path.abspath('./../'))
 autodoc_mock_imports = ["alpha", "beta"]
 
 
 # import sphinx.apidoc
 def setup(app):
+    src = os.path.join(_here, '..', 'src')
     sphinx.ext.apidoc.main(
-        ['-f', '-T', '-e', '-M', '-o', './code/', '../src', '../src/localconfig.py', '../src/pyff/test']
+        [
+            '-f',
+            '-T',
+            '-e',
+            '-M',
+            '-o',
+            os.path.join(_here, 'code'),
+            src,
+            os.path.join(src, 'localconfig.py'),
+            os.path.join(src, 'pyff', 'test'),
+        ]
     )
 
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('../src'))
+sys.path.insert(0, os.path.join(_here, '..', 'src'))
+sys.path.insert(0, os.path.join(_here, '_ext'))
 
 # -- General configuration -----------------------------------------------------
 
@@ -38,7 +52,7 @@ def setup(app):
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.apidoc', 'sphinx.ext.autosummary']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.apidoc', 'sphinx.ext.autosummary', 'pyff_pipes']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

docs/index.rst

@@ -34,6 +34,7 @@ transforming metadata to add custom elements.
    usage/running
    usage/deploying
    usage/examples
+   usage/pipes
    coding
    faq
 

docs/usage/pipes.rst

@@ -0,0 +1,16 @@
+Pipes
+=====
+
+A pipeline is a YAML list of steps, called pipes. Each step is the name of a pipe, optionally
+followed by options on the same line and arguments after the colon:
+
+.. code-block:: yaml
+
+    - load fail_on_error True:
+        - http://example.com/metadata.xml
+    - select
+    - publish: /tmp/metadata.xml
+
+This page is generated from the docstrings of the pipes in :py:mod:`pyff.builtins`.
+
+.. pyff-pipes::