From e5ac0f93dac4772a73ef8cfd4fe7d9ce0d9e9755 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 22 Oct 2026 16:16:09 +0200 Subject: [PATCH 1/2] Improve documentation Closes #305 --- .readthedocs.yml | 11 ++- docs/_ext/pyff_pipes.py | 56 ++++++++++++++ docs/conf.py | 20 ++++- docs/index.rst | 1 + docs/usage/pipes.rst | 16 ++++ src/pyff/builtins.py | 168 +++++++++++++++++++++++++--------------- 6 files changed, 203 insertions(+), 69 deletions(-) create mode 100644 docs/_ext/pyff_pipes.py create mode 100644 docs/usage/pipes.rst diff --git a/.readthedocs.yml b/.readthedocs.yml index 20d3d09e..88aacc3f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,11 +1,14 @@ version: 2 sphinx: configuration: docs/conf.py -formats: all +formats: all build: - os: "ubuntu-20.04" + os: "ubuntu-24.04" tools: - python: "3.8" + python: "3.13" python: install: - - requirements: docs/requirements.txt + - method: uv + command: sync + groups: + - docs diff --git a/docs/_ext/pyff_pipes.py b/docs/_ext/pyff_pipes.py new file mode 100644 index 00000000..e304b9a4 --- /dev/null +++ b/docs/_ext/pyff_pipes.py @@ -0,0 +1,56 @@ +""" +Sphinx directive that documents every registered pyFF pipe from its docstring. + +Usage in a document:: + + .. pyff-pipes:: + +Pipes are listed by the name used in pipeline YAML (the key in +:py:data:`pyff.pipes.registry`), not by the name of the Python function. +""" + +import inspect +import re + +from docutils import nodes +from docutils.statemachine import StringList +from sphinx.util.docutils import SphinxDirective +from sphinx.util.nodes import nested_parse_with_titles + +# Field lines describe the Python signature, which is irrelevant for pipeline authors. +_PYTHON_FIELDS = re.compile(r'^\s*:(param|type|return|returns|rtype)\b') + + +def _pipe_docs(): + import pyff.builtins # noqa: F401 (importing registers the pipes) + from pyff.pipes import registry + + for name, func in sorted(registry.items()): + doc = inspect.getdoc(func) + # Deprecated pipes are wrapped and end up without a docstring. + if doc: + yield name, doc + + +def _strip_python_fields(doc): + return '\n'.join(line for line in doc.splitlines() if not _PYTHON_FIELDS.match(line)) + + +class PyffPipes(SphinxDirective): + has_content = False + + def run(self): + lines = [] + for name, doc in _pipe_docs(): + lines += [name, '-' * len(name), '', _strip_python_fields(doc), ''] + + node = nodes.section() + node.document = self.state.document + content = StringList('\n'.join(lines).splitlines(), source=f'<{self.name}>') + nested_parse_with_titles(self.state, content, node) + return node.children + + +def setup(app): + app.add_directive('pyff-pipes', PyffPipes) + return {'parallel_read_safe': True, 'parallel_write_safe': True} diff --git a/docs/conf.py b/docs/conf.py index 2e5193e2..cb87c1fe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,21 +15,35 @@ import sphinx +_here = os.path.dirname(os.path.abspath(__file__)) + sys.path.insert(0, os.path.abspath('./../')) autodoc_mock_imports = ["alpha", "beta"] # import sphinx.apidoc def setup(app): + src = os.path.join(_here, '..', 'src') sphinx.ext.apidoc.main( - ['-f', '-T', '-e', '-M', '-o', './code/', '../src', '../src/localconfig.py', '../src/pyff/test'] + [ + '-f', + '-T', + '-e', + '-M', + '-o', + os.path.join(_here, 'code'), + src, + os.path.join(src, 'localconfig.py'), + os.path.join(src, 'pyff', 'test'), + ] ) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('../src')) +sys.path.insert(0, os.path.join(_here, '..', 'src')) +sys.path.insert(0, os.path.join(_here, '_ext')) # -- General configuration ----------------------------------------------------- @@ -38,7 +52,7 @@ def setup(app): # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.apidoc', 'sphinx.ext.autosummary'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.apidoc', 'sphinx.ext.autosummary', 'pyff_pipes'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/index.rst b/docs/index.rst index 2257b25b..62d7e5ec 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,6 +34,7 @@ transforming metadata to add custom elements. usage/running usage/deploying usage/examples + usage/pipes coding faq diff --git a/docs/usage/pipes.rst b/docs/usage/pipes.rst new file mode 100644 index 00000000..37f89ffd --- /dev/null +++ b/docs/usage/pipes.rst @@ -0,0 +1,16 @@ +Pipes +===== + +A pipeline is a YAML list of steps, called pipes. Each step is the name of a pipe, optionally +followed by options on the same line and arguments after the colon: + +.. code-block:: yaml + + - load fail_on_error True: + - http://example.com/metadata.xml + - select + - publish: /tmp/metadata.xml + +This page is generated from the docstrings of the pipes in :py:mod:`pyff.builtins`. + +.. pyff-pipes:: diff --git a/src/pyff/builtins.py b/src/pyff/builtins.py index 71621b1c..39f0824c 100644 --- a/src/pyff/builtins.py +++ b/src/pyff/builtins.py @@ -111,7 +111,7 @@ def _map(req: Plumbing.Request, *opts): - map: - ...statements... - Executes a set of statements in parallell (using a thread pool). + Executes a set of statements in parallel (using a thread pool). """ @@ -163,8 +163,10 @@ def _print_t(req: Plumbing.Request, *opts): .. code-block:: yaml - - print - output: "somewhere.foo" + - print: + output: "somewhere.foo" + + Without ``output`` the tree is printed on stdout. """ fn = None @@ -192,6 +194,14 @@ def end(req: Plumbing.Request, *opts): - end - unreachable + The optional ``code`` and ``message`` arguments set the exit code and a message printed on stdout: + + .. code-block:: yaml + + - end: + code: 1 + message: "no metadata to publish" + **Warning** This is very bad if used with pyffd - the server will stop running. If you just want to break out of the pipeline, use break instead. @@ -208,7 +218,7 @@ def end(req: Plumbing.Request, *opts): @pipe def fork(req: Plumbing.Request, *opts): """ - Make a copy of the working tree and process the arguments as a pipleline. This essentially resets the working + Make a copy of the working tree and process the arguments as a pipeline. This essentially resets the working tree and allows a new plumbing to run. Useful for producing multiple outputs from a single source. :param req: The request @@ -226,7 +236,7 @@ def fork(req: Plumbing.Request, *opts): output: "/tmp/annotated.xml" - fork: - xslt: - stylesheet: tidy.xml + stylesheet: tidy.xsl - publish: output: "/tmp/clean.xml" @@ -235,7 +245,7 @@ def fork(req: Plumbing.Request, *opts): **Merging** - Normally the result of the "inner" plumbing is disgarded - unless published or emit:ed to a calling client + Normally the result of the "inner" plumbing is discarded - unless published or emit:ed to a calling client in the case of the MDX server - but by adding 'merge' to the options with an optional 'merge strategy' the behaviour can be changed to merge the result of the inner pipeline back to the parent working document. @@ -280,7 +290,7 @@ def fork(req: Plumbing.Request, *opts): output: "/tmp/annotated.xml" - fork: - xslt: - stylesheet: tidy.xml + stylesheet: tidy.xsl - publish: output: "/tmp/clean.xml" """ @@ -351,7 +361,7 @@ def _break(req: Plumbing.Request, *opts): @pipe(name='pipe') def _pipe(req: Plumbing.Request, *opts): """ - Run the argument list as a pipleine. + Run the argument list as a pipeline. :param req: The request :param opts: Options (unused) @@ -413,9 +423,9 @@ def when(req: Plumbing.Request, condition: str, *values): .. code-block:: yaml - - when foo + - when foo: - something - - when bar bill + - when bar bill: - other The condition operates on the state: if 'foo' is present in the state (with any value), then the something branch is @@ -459,17 +469,18 @@ def sort(req: Plumbing.Request, *opts): Entities where no value exists for a given xpath are sorted last. :param req: The request - :param opts: Options: (see bellow) + :param opts: Options: (see below) :return: None Options are put directly after "sort". E.g: .. code-block:: yaml - - sort order_by [xpath] + - sort order_by ./md:IDPSSODescriptor/md:Extensions/mdui:UIInfo/mdui:DisplayName **Options** - - order_by [xpath] : xpath expression selecting to the value used for sorting the entities. + + - ``order_by ``: xpath expression selecting the value used for sorting the entities. """ if req.t is None: raise PipeException("Unable to sort empty document.") @@ -490,7 +501,8 @@ def publish(req: Plumbing.Request, *opts): :param opts: Options (unused) :return: None - Publish takes one argument: path to a file where the document tree will be written. + Publish takes one argument: path to a file where the document tree will be written. + **Examples** .. code-block:: yaml @@ -636,7 +648,12 @@ def load(req: Plumbing.Request, *opts): Supports both remote and local resources. Fetching remote resources is done in parallel using threads. - Note: When downloading remote files over HTTPS the TLS server certificate is not validated by default + .. warning:: + + When downloading remote files over HTTPS the TLS server certificate is **not** validated by default + (``verify_tls`` is False). Metadata signatures are only checked if a verification is given per + resource, see below. + Note: Default behaviour is to ignore metadata files or entities in MD files that cannot be loaded Options are put directly after "load". E.g: @@ -649,19 +666,36 @@ def load(req: Plumbing.Request, *opts): - /opt/directory_containing_md_files/ **Options** - Defaults are marked with (*) - - max_workers <5> : Number of parallel threads to use for loading MD files - - timeout <120> : Socket timeout when downloading files - - validate : When true downloaded metadata files are validated (schema validation) - - fail_on_error : Control whether an error during download, parsing or (optional)validation of a MD - file does not abort processing of the pipeline. When true a failure aborts and - causes pyff to exit with a non zero exit code. Otherwise errors are logged but - ignored. - - filter_invalid : Controls validation behaviour. When true Entities that fail validation are filtered - I.e. are not loaded. When false the entire metadata file is either loaded, or not. - fail_on_error controls whether failure to validating the entire MD file will abort - processing of the pipeline. - - verify_tls : Controls the validation of the host's TLS certificate on fetching the resources + + Defaults are marked with (*). + + - ``max_workers <5>``: Number of parallel threads to use for loading MD files + - ``timeout <120>``: Socket timeout (seconds) when downloading files + - ``validate ``: When true downloaded metadata files are validated (schema validation) + - ``fail_on_error ``: Control whether an error during download, parsing or (optional) validation + of a MD file aborts processing of the pipeline. When true a failure aborts and causes pyff to exit with a + non zero exit code. Otherwise errors are logged but ignored. + - ``filter_invalid ``: Controls validation behaviour. When true entities that fail validation + are filtered, i.e. are not loaded. When false the entire metadata file is either loaded, or not. + ``fail_on_error`` controls whether failure to validate the entire MD file will abort processing of the + pipeline. + - ``verify_tls ``: Controls the validation of the host's TLS certificate on fetching the resources + + **Resources** + + Each resource in the list has the form + + .. code-block:: text + + [as ] [verify ] [via ]* [cleanup ]* + + - ``as``: name the resource, e.g. to refer to it in later pipeline steps. + - ``verify``: check the XML signature of the resource, using either a PEM certificate file or a SHA-1 + fingerprint of the signing certificate. With a fingerprint, the certificate embedded in the signature is + used and the fingerprint pins it. + - ``via``: name of a ``when`` clause in the same pipeline that is applied to the loaded metadata, for + example to remove entities with ``fork merge remove``. + - ``cleanup``: name of a ``when`` clause used to pre-process the parsed metadata before it is validated. """ _opts = dict(list(zip(opts[::2], opts[1::2]))) _opts.setdefault('timeout', 120) @@ -759,8 +793,7 @@ def select(req: Plumbing.Request, *opts): :return: returns the result of the operation as a working document Select picks and expands elements (with optional filtering) from the active repository you setup using calls - to :py:mod:`pyff.pipes.builtins.load`. See :py:mod:`pyff.mdrepo.MDRepository.lookup` for a description of the syntax - for selectors. + to ``load``. See :py:meth:`pyff.repo.MDRepository.lookup` for a description of the syntax for selectors. **Examples** @@ -774,8 +807,8 @@ def select(req: Plumbing.Request, *opts): - select: "/var/local-metadata" - This would select all entities found in the directory /var/local-metadata. You must have a call to local to load - entities from this directory before select statement. + This would select all entities found in the directory /var/local-metadata. You must have a call to ``load`` to + load entities from this directory before the select statement. .. code-block:: yaml @@ -792,7 +825,7 @@ def select(req: Plumbing.Request, *opts): Select statements are not cumulative - a select followed by another select in the plumbing resets the working documents to the result of the second select. - Most statements except local and remote depend on having a select somewhere in your plumbing and will + Most statements except ``load`` depend on having a select somewhere in your plumbing and will stop the plumbing if the current working document is empty. For instance, running .. code-block:: yaml @@ -809,22 +842,25 @@ def select(req: Plumbing.Request, *opts): - select as /foo-2.0 dedup True: "!//md:EntityDescriptor[md:IDPSSODescriptor]" **Options** - Defaults are marked with (*) - - as : The 'as' keyword allows a select to be stored as an alias in the local repository. For instance - .. code-block:: yaml + Defaults are marked with (*). + + - ``as ``: The 'as' keyword allows a select to be stored as an alias in the local repository. + For instance - - select as /foo-2.0: "!//md:EntityDescriptor[md:IDPSSODescriptor]" + .. code-block:: yaml - would allow you to use /foo-2.0.json to refer to the JSON-version of all IdPs in the current repository. - Note that you should not include an extension in your "as foo-bla-something" since that would make your - alias invisible for anything except the corresponding mime type. + - select as /foo-2.0: "!//md:EntityDescriptor[md:IDPSSODescriptor]" - - dedup : Whether to deduplicate the results by entityID. + would allow you to use /foo-2.0.json to refer to the JSON-version of all IdPs in the current repository. + Note that you should not include an extension in your "as foo-bla-something" since that would make your + alias invisible for anything except the corresponding mime type. - Note: When select is used after a load pipe with more than one source, if dedup is set to True - and there are entity properties that may differ from one source to another, these will be squashed - rather than merged. + - ``dedup ``: Whether to deduplicate the results by entityID. + + Note: When select is used after a load pipe with more than one source, if dedup is set to True + and there are entity properties that may differ from one source to another, these will be squashed + rather than merged. """ opt_names = ('as', 'dedup') if len(opts) % 2 == 0: @@ -974,7 +1010,7 @@ def pick(req: Plumbing.Request, *opts): :param opts: Options (unused) :return: returns the result of the operation as a working document - Useful for testing. See py:mod:`pyff.pipes.builtins.pick` for more information about selecting the document. + Useful for testing. See ``select`` for more information about selecting the document. """ args = _select_args(req) @@ -1021,7 +1057,8 @@ def _discojson(req: Plumbing.Request, *opts): Return a discojuice-compatible json representation of the tree .. code-block:: yaml - discojson: + + - discojson If the config.load_icons directive is set the icons will be returned from a (possibly persistent) local cache & converted to data: URIs @@ -1048,7 +1085,8 @@ def _discojson_sp(req, *opts): Return a json representation of the trust information .. code-block:: yaml - discojson_sp: + + - discojson_sp The returned json doc will have the following structure. @@ -1109,7 +1147,8 @@ def _discojson_sp_attr(req, *opts): Return a json representation of the trust information .. code-block:: yaml - discojson_sp_attr: + + - discojson_sp_attr SP Entities can carry trust information as a base64 encoded json blob as an entity attribute with name `https://refeds.org/entity-selection-profile`. @@ -1506,7 +1545,7 @@ def xslt(req: Plumbing.Request, *opts): .. code-block:: yaml - xslt: - sylesheet: foo.xsl + stylesheet: foo.xsl x: foo y: bar @@ -1599,7 +1638,8 @@ def prune(req: Plumbing.Request, *opts): :return: The tree with all specified elements removed - ** Examples** + **Examples** + .. code-block:: yaml - prune: @@ -1612,7 +1652,7 @@ def prune(req: Plumbing.Request, *opts): - prune: - .//{http://www.w3.org/2000/09/xmldsig#}Signature[1] - This example would drop the first Signature element only. + This example would drop the first Signature element below each parent element. """ @@ -1636,7 +1676,8 @@ def prune(req: Plumbing.Request, *opts): @pipe def check_xml_namespaces(req: Plumbing.Request, *opts): """ - Ensure that all namespaces are http or httpd scheme URLs. + Ensure that all namespaces are http, https or urn URIs. A namespace URI with any other scheme causes a + MetadataException. :param req: The request :param opts: Options (not used) @@ -1669,7 +1710,7 @@ def check_xml_namespaces(req: Plumbing.Request, *opts): @pipe def drop_xsi_type(req: Plumbing.Request, *opts): """ - Remove all xsi namespaces from the tree. + Remove all ``xsi:type`` attributes from the tree. :param req: The request :param opts: Options (not used) @@ -1919,7 +1960,7 @@ def finalize(req: Plumbing.Request, *opts): For instance +45d 2m results in a time delta of 45 days and 2 minutes. The '+' sign is optional. - If operating on a single EntityDescriptor then @Name is ignored (cf :py:mod:`pyff.pipes.builtins.first`). + If operating on a single EntityDescriptor then @Name is ignored (cf ``first``). **Examples** @@ -2025,9 +2066,11 @@ def _reginfo(req: Plumbing.Request, *opts): .. code-block:: yaml - reginfo: - [policy: - : ] - authority: + authority: http://registrar.example.org/ + policy: + en: http://registrar.example.org/policy.html + + ``policy`` is optional and maps a language code to the registration policy URL. """ if req.t is None: @@ -2051,15 +2094,17 @@ def _pubinfo(req: Plumbing.Request, *opts): :param opts: Options (not used) :return: A modified working document - Transforms the working document by setting the specified attribute on all of the EntityDescriptor - elements of the active document. + Transforms the working document by adding a PublicationInfo extension to its top-level element. An error is + raised if one is already present. **Examples** .. code-block:: yaml - pubinfo: - publisher: + publisher: http://publisher.example.org/ + + ``creation_instant`` (an ISO 8601 time string) is optional and defaults to the current time. """ if req.t is None: @@ -2092,7 +2137,6 @@ def _setattr(req: Plumbing.Request, *opts): - setattr: attr1: value1 attr2: value2 - ... Normally this would be combined with the 'merge' feature of fork to add attributes to the working document for later processing. @@ -2126,7 +2170,7 @@ def _nodecountry(req: Plumbing.Request, *opts): - nodecountry: country: XX - Normally this would be combined with the 'merge' feature of fork or in a cleanup pipline to add attributes to + Normally this would be combined with the 'merge' feature of fork or in a cleanup pipeline to add attributes to the working document for later processing. """ From 543a919c4bc5e8e9055351e9e0e1c80a4a112bae Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 22 Oct 2026 16:18:59 +0200 Subject: [PATCH 2/2] Fix ip_adress tyop in select match This code path has been broken since 2021. --- src/pyff/builtins.py | 2 +- src/pyff/test/test_pipeline.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/pyff/builtins.py b/src/pyff/builtins.py index 39f0824c..710f93a1 100644 --- a/src/pyff/builtins.py +++ b/src/pyff/builtins.py @@ -922,7 +922,7 @@ def _match(q, elt): try: nets = _ip_networks(elt) for net in nets: - if ipaddress.ip_adress(q) in net: + if ipaddress.ip_address(q) in net: return net except ValueError: pass diff --git a/src/pyff/test/test_pipeline.py b/src/pyff/test/test_pipeline.py index dca53928..9e5a3323 100644 --- a/src/pyff/test/test_pipeline.py +++ b/src/pyff/test/test_pipeline.py @@ -20,6 +20,7 @@ from pyff.pipes import PipeException, Plumbing, plumbing from pyff.repo import MDRepository from pyff.resource import ResourceException +from pyff.samlmd import iter_entities from pyff.test import ExitException, SignerTestCase from pyff.utils import hash_id, parse_xml, resource_filename, root @@ -106,6 +107,20 @@ def test_parse(self): in self.captured_log_text ) + def test_select_match_ip_hint(self): + md = MDRepository() + p = yaml.safe_load( + StringIO( + f""" +- load: + - {self.datadir}/metadata/test01.xml +- select +""" + ) + ) + res = Plumbing(p, pid="test").process(md, state={'batch': True, 'stats': {}, 'match': '10.1.2.3'}) + assert [e.get('entityID') for e in iter_entities(res)] == ['https://idp.example.com/saml2/idp/metadata.php'] + # To run all LoadErrorTests: ./setup.py test -s pyff.test.test_pipeline.LoadErrorTest # To run individual test: ./setup.py test -s pyff.test.test_pipeline.LoadErrorTest.test_fail_on_error_no_file