diff --git a/databusclient/api/download.py b/databusclient/api/download.py index a4cf7dc..f296fd7 100644 --- a/databusclient/api/download.py +++ b/databusclient/api/download.py @@ -1184,6 +1184,7 @@ def _download_artifact( uri: str, localDir: str, all_versions: bool = False, + version: str | None = None, vault_token_file: str = None, databus_key: str = None, auth_url: str = None, @@ -1203,6 +1204,7 @@ def _download_artifact( uri: The full databus artifact URI. localDir: Local directory to download files to. If None, the databus folder structure is created in the current working directory. all_versions: If True, download all versions of the artifact; otherwise, only download the latest version. + version: Optional version selector ('latest', 'all', or specific version ID/URI). vault_token_file: Path to Vault refresh token file for protected downloads. databus_key: Databus API key for protected downloads. auth_url: Keycloak token endpoint URL. @@ -1217,7 +1219,9 @@ def _download_artifact( the versioned databus-python-client default. """ json_str = fetch_databus_jsonld(uri, databus_key=databus_key) - versions = _get_databus_versions_of_artifact(json_str, all_versions=all_versions) + versions = _get_databus_versions_of_artifact( + json_str, all_versions=all_versions, version=version + ) if isinstance(versions, str): versions = [versions] for version_uri in versions: @@ -1251,17 +1255,21 @@ def _download_artifact( def _get_databus_versions_of_artifact( - json_str: str, all_versions: bool + json_str: str, + all_versions: bool = False, + version: str | None = None, ) -> str | List[str]: """Parse the JSON-LD of a databus artifact to extract URLs of its versions. Args: json_str: JSON-LD string of the databus artifact. all_versions: If True, return all version URLs; otherwise, return only the latest version URL. + version: Optional version selector: 'latest', 'all', or a specific version ID / URI. Returns: - If all_versions is True: List of all version URLs. - If all_versions is False: URL of the latest version. + If all_versions is True or version is 'all': List of all version URLs. + If version is a specific identifier: URL of the matching version. + If all_versions is False and version is None or 'latest': URL of the latest version. """ json_dict = json.loads(json_str) versions = json_dict.get("databus:hasVersion") @@ -1283,6 +1291,29 @@ def _get_databus_versions_of_artifact( version_urls.sort(reverse=True) # Sort versions in descending order + if version is not None: + v_clean = version.strip() + if v_clean.lower() == "all": + return version_urls + if v_clean.lower() == "latest": + return version_urls if all_versions else version_urls[0] + + if "://" in v_clean: + matched = [u for u in version_urls if u.rstrip("/") == v_clean.rstrip("/")] + else: + target_id = v_clean.rstrip("/") + matched = [ + u + for u in version_urls + if u.rstrip("/").split("/")[-1] == target_id + ] + if not matched: + available = ", ".join(u.rstrip("/").split("/")[-1] for u in version_urls) + raise ValueError( + f"Version '{version}' not found for artifact. Available versions: {available}" + ) + return matched[0] + if all_versions: return version_urls return version_urls[0] @@ -1315,6 +1346,7 @@ def _download_group( uri: str, localDir: str, all_versions: bool = False, + version: str | None = None, vault_token_file: str = None, databus_key: str = None, auth_url: str = None, @@ -1334,6 +1366,7 @@ def _download_group( uri: The full databus group URI. localDir: Local directory to download files to. If None, the databus folder structure is created in the current working directory. all_versions: If True, download all versions of each artifact in the group; otherwise, only download the latest version. + version: Optional version selector ('latest', 'all', or specific version ID/URI). vault_token_file: Path to Vault refresh token file for protected downloads. databus_key: Databus API key for protected downloads. auth_url: Keycloak token endpoint URL. @@ -1351,23 +1384,38 @@ def _download_group( artifacts = _get_databus_artifacts_of_group(json_str) for artifact_uri in artifacts: print(f"Download artifact: {artifact_uri}") - _download_artifact( - artifact_uri, - localDir, - all_versions=all_versions, - vault_token_file=vault_token_file, - databus_key=databus_key, - auth_url=auth_url, - client_id=client_id, - compression=compression, - convert_format=convert_format, - graph_name=graph_name, - base_uri=base_uri, - graph_mode=graph_mode, - manifest_context=manifest_context, - validate_checksum=validate_checksum, - user_agent=user_agent, - ) + try: + _download_artifact( + artifact_uri, + localDir, + all_versions=all_versions, + version=version, + vault_token_file=vault_token_file, + databus_key=databus_key, + auth_url=auth_url, + client_id=client_id, + compression=compression, + convert_format=convert_format, + graph_name=graph_name, + base_uri=base_uri, + graph_mode=graph_mode, + manifest_context=manifest_context, + validate_checksum=validate_checksum, + user_agent=user_agent, + ) + except ValueError as err: + # If a specific version was requested and is not found for this artifact, log and continue with others + if ( + version is not None + and version.strip().lower() not in ("latest", "all") + and ( + "not found for artifact" in str(err) + or "No versions found in artifact JSON-LD" in str(err) + ) + ): + print(f"Skipping artifact {artifact_uri}: {err}") + continue + raise def _get_databus_artifacts_of_group(json_str: str) -> List[str]: @@ -1421,6 +1469,8 @@ def download( validate_checksum: bool = False, manifest_context=None, user_agent: str | None = None, + version: str | None = None, + target_version: str | None = None, ) -> None: """Download datasets from databus. @@ -1432,6 +1482,7 @@ def download( databusURIs: Databus identifiers to specify datasets to download. token: Path to Vault refresh token file for protected downloads. databus_key: Databus API key for protected downloads. + all_versions: If True, download all versions of artifacts in group/artifact URIs. auth_url: Keycloak token endpoint URL. Default is "https://auth.dbpedia.org/realms/dbpedia/protocol/openid-connect/token". client_id: Client ID for token exchange. Default is "vault-token-exchange". compression: Target compression format for on-the-fly conversion (supported: bz2, gz, xz). @@ -1441,12 +1492,21 @@ def download( base_uri: Base URI for CSV -> Triple conversion (Layer 3). graph_mode: Optional graph sidecar mode. Currently supports 'download-url'. validate_checksum: Whether to validate checksums after downloading. + manifest_context: Optional context for recording operation manifest. user_agent: HTTP User-Agent for file HEAD and GET requests. None uses the versioned databus-python-client default. + version: Target version selector ('latest', 'all', or specific version ID/URI). + target_version: Alias for version. """ _validate_graph_mode(graph_mode) + resolved_version = target_version if target_version is not None else version + if resolved_version and resolved_version.strip().lower() == "all": + all_versions = True + elif all_versions and resolved_version is None: + resolved_version = "all" + for databusURI in databusURIs: - host, account, group, artifact, version, file = get_databus_id_parts_from_file_url(databusURI) + host, account, group, artifact, uri_version, file = get_databus_id_parts_from_file_url(databusURI) # Determine endpoint per-URI if not explicitly provided uri_endpoint = endpoint @@ -1504,7 +1564,7 @@ def download( expected_checksum=expected, user_agent=user_agent, ) - elif version is not None: + elif uri_version is not None: print(f"Downloading version: {databusURI}") _download_version( databusURI, @@ -1523,13 +1583,19 @@ def download( user_agent=user_agent, ) elif artifact is not None: + v_desc = ( + resolved_version + if resolved_version and resolved_version.strip().lower() not in ("latest", "all") + else ("all" if all_versions else "latest") + ) print( - f"Downloading {'all' if all_versions else 'latest'} version(s) of artifact: {databusURI}" + f"Downloading {v_desc} version(s) of artifact: {databusURI}" ) _download_artifact( databusURI, localDir, - all_versions=all_versions, + all_versions=bool(all_versions), + version=resolved_version, vault_token_file=token, databus_key=databus_key, auth_url=auth_url, @@ -1544,13 +1610,19 @@ def download( user_agent=user_agent, ) elif group is not None and group != "collections": + v_desc = ( + resolved_version + if resolved_version and resolved_version.strip().lower() not in ("latest", "all") + else ("all" if all_versions else "latest") + ) print( - f"Downloading group and all its artifacts and versions: {databusURI}" + f"Downloading group ({v_desc} versions): {databusURI}" ) _download_group( databusURI, localDir, - all_versions=all_versions, + all_versions=bool(all_versions), + version=resolved_version, vault_token_file=token, databus_key=databus_key, auth_url=auth_url, diff --git a/databusclient/cli.py b/databusclient/cli.py index 2fd58a3..9c3618c 100644 --- a/databusclient/cli.py +++ b/databusclient/cli.py @@ -250,6 +250,13 @@ def _write_manifest(): is_flag=True, help="When downloading artifacts, download all versions instead of only the latest", ) +@click.option( + "--version", + "--target-version", + "target_version", + default=None, + help="Target version to download: 'latest' (default), 'all', or a specific version ID (e.g. 2025-10-14).", +) @click.option( "--authurl", default="https://auth.dbpedia.org/realms/dbpedia/protocol/openid-connect/token", @@ -332,6 +339,7 @@ def download( vault_token, databus_key, all_versions, + target_version, authurl, clientid, compression, @@ -347,6 +355,11 @@ def download( Download datasets from databus, optionally using vault access if vault options are provided. Supports on-the-fly compression format conversion using the --compression option. """ + if all_versions and not target_version: + target_version = "all" + elif target_version and target_version.strip().lower() == "all": + all_versions = True + # Determine auth method for manifest (never store the token itself) auth_method = None if vault_token: @@ -370,6 +383,7 @@ def download( "base_uri": base_uri, "graph_mode": graph_mode, "all_versions": all_versions, + "version": target_version, "validate_checksum": validate_checksum, "user_agent": user_agent, "authurl": authurl, @@ -383,6 +397,7 @@ def download( token=vault_token, databus_key=databus_key, all_versions=all_versions, + version=target_version, auth_url=authurl, client_id=clientid, compression=compression, diff --git a/databusclient/manifest/replay.py b/databusclient/manifest/replay.py index 4def7bc..a39927a 100644 --- a/databusclient/manifest/replay.py +++ b/databusclient/manifest/replay.py @@ -42,6 +42,16 @@ def _build_download_kwargs( replay_params: Dict[str, Any], overrides: Dict[str, Any], ) -> Dict[str, Any]: + """Build keyword arguments for api_download from manifest replay parameters. + + Args: + manifest: Operation manifest dictionary. + replay_params: Parameters recorded during the original operation. + overrides: Execution overrides provided at replay time. + + Returns: + Keyword argument dictionary ready for api_download. + """ databus_uris = replay_params.get("databusURIs") if not isinstance(databus_uris, list) or not databus_uris: raise ManifestReplayError( @@ -63,6 +73,12 @@ def _build_download_kwargs( endpoint = overrides.get("endpoint", manifest.get("dbus:endpoint")) + version = replay_params.get("version") + if version is not None and not isinstance(version, str): + raise ManifestReplayError( + "Manifest field dbus:replayParams.version must be a string." + ) + return { "localDir": overrides.get("localDir"), "endpoint": endpoint, @@ -70,6 +86,7 @@ def _build_download_kwargs( "token": token, "databus_key": databus_key, "all_versions": replay_params.get("all_versions"), + "version": version, "auth_url": replay_params.get( "authurl", "https://auth.dbpedia.org/realms/dbpedia/protocol/openid-connect/token", diff --git a/databusclient/workflow/steps.py b/databusclient/workflow/steps.py index 320e616..884bd7d 100644 --- a/databusclient/workflow/steps.py +++ b/databusclient/workflow/steps.py @@ -76,6 +76,7 @@ def run(self, step_config: Dict[str, Any], context: StepContext) -> None: token=resolved.get("vault_token"), databus_key=resolved.get("databus_key"), all_versions=resolved.get("all_versions", False), + version=resolved.get("version"), compression=resolved.get("convert_to") or resolved.get("compression"), convert_format=resolved.get("format"), graph_name=resolved.get("graph_name"), diff --git a/doc/cli-usage.md b/doc/cli-usage.md index 2682648..3258f28 100644 --- a/doc/cli-usage.md +++ b/doc/cli-usage.md @@ -1,4 +1,4 @@ -## CLI Usage +## CLI Usage To get started with the command-line interface (CLI) of the databus-python-client, you can use either the Python installation or the Docker image. The examples below show python installation method. @@ -28,8 +28,13 @@ databusclient download $DOWNLOADTARGET Note: Vault tokens are only required for certain protected Databus hosts (for example: `data.dbpedia.io`, `data.dev.dbpedia.link`). The client now detects those hosts and will fail early with a clear message if a token is required but not provided. Do not pass `--vault-token` for public downloads. - `--databus-key` - If the databus is protected and needs API key authentication, you can provide the API key with `--databus-key YOUR_API_KEY`. +- `--version`, `--target-version` + - Specifies which version(s) to download when downloading an artifact or group: + - `--version latest`: downloads only the latest version (default behavior). + - `--version all`: downloads all versions of artifacts (equivalent to `--all-versions`). + - `--version `: downloads a specific version if available (e.g. `--version 2022.12.01`). - `--all-versions` - - When downloading artifacts, downloads all versions instead of only the latest. + - When downloading artifacts, downloads all versions instead of only the latest (equivalent to `--version all`). - `--compression` - Enables on-the-fly compression format conversion during download. Supported formats: `bz2`, `gz`, `xz`, `none`. The source compression is auto-detected from the file extension. Use `none` to decompress files without recompressing. Example: `--compression gz` converts all downloaded compressed files to gzip format. - `--format` @@ -65,6 +70,16 @@ databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals ``` +**Download Specific Version of Artifact or Group**: download a specific version using `--version` (or `--target-version`) +```bash +databusclient download --version 2022.12.01 https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals +``` + +**Download All Versions of Artifact or Group**: download all historical versions using `--version all` or `--all-versions` +```bash +databusclient download --version all https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals +``` + **Download Group**: download of all files with the latest version of all artifacts of a group ```bash databusclient download https://databus.dbpedia.org/dbpedia/mappings diff --git a/doc/module-usage.md b/doc/module-usage.md index 68a8282..1e88710 100644 --- a/doc/module-usage.md +++ b/doc/module-usage.md @@ -80,3 +80,35 @@ deploy(dataset, "mysterious API key") ``` The API key can be found or generated in the Databus account settings. + + +### Download + +You can also use the download function directly in Python: + +```python +from databusclient.api.download import download + +# Download latest version of an artifact +download( + localDir="./data", + endpoint=None, + databusURIs=["https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals"], +) + +# Download a specific version of an artifact or group +download( + localDir="./data", + endpoint=None, + databusURIs=["https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals"], + version="2022.12.01", +) + +# Download all versions of an artifact or group +download( + localDir="./data", + endpoint=None, + databusURIs=["https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals"], + version="all", +) +``` diff --git a/tests/test_download_version_selection.py b/tests/test_download_version_selection.py new file mode 100644 index 0000000..214efca --- /dev/null +++ b/tests/test_download_version_selection.py @@ -0,0 +1,468 @@ +"""Tests for version selection in downloads (Issue #17).""" + +import json +import pytest +from click.testing import CliRunner + +import databusclient.api.download as dl +from databusclient.api.download import ( + _get_databus_versions_of_artifact, + download as api_download, +) +from databusclient.cli import app +from databusclient.manifest.replay import ManifestReplayError, replay_manifest +from databusclient.workflow.steps import DownloadStep +from databusclient.workflow.context import StepContext + + +SAMPLE_ARTIFACT_JSONLD = json.dumps({ + "@id": "https://databus.dbpedia.org/user/group/my-artifact", + "@type": "dataid:Artifact", + "databus:hasVersion": [ + {"@id": "https://databus.dbpedia.org/user/group/my-artifact/2021.01.01"}, + {"@id": "https://databus.dbpedia.org/user/group/my-artifact/2022.05.01"}, + {"@id": "https://databus.dbpedia.org/user/group/my-artifact/2023.10.15"}, + ], +}) + +SAMPLE_VERSION_JSONLD = json.dumps({ + "@id": "https://databus.dbpedia.org/user/group/my-artifact/2022.05.01", + "@type": "dataid:Version", + "@graph": [ + { + "@type": "Part", + "file": "https://example.org/files/data_20220501.ttl", + } + ], +}) + +SAMPLE_GROUP_JSONLD = json.dumps({ + "@id": "https://databus.dbpedia.org/user/group", + "@type": "dataid:Group", + "databus:hasArtifact": [ + {"@id": "https://databus.dbpedia.org/user/group/artifact-one"}, + {"@id": "https://databus.dbpedia.org/user/group/artifact-two"}, + ], +}) + + +# --------------------------------------------------------------------------- +# Unit tests: _get_databus_versions_of_artifact +# --------------------------------------------------------------------------- + +def test_get_versions_default_returns_latest(): + """Verify default call without version or all_versions returns latest version URL.""" + res = _get_databus_versions_of_artifact(SAMPLE_ARTIFACT_JSONLD) + assert res == "https://databus.dbpedia.org/user/group/my-artifact/2023.10.15" + + +def test_get_versions_explicit_latest(): + """Verify version='latest' returns latest version URL.""" + res = _get_databus_versions_of_artifact(SAMPLE_ARTIFACT_JSONLD, version="latest") + assert res == "https://databus.dbpedia.org/user/group/my-artifact/2023.10.15" + + +def test_get_versions_version_latest_with_all_versions_flag(): + """Verify all_versions=True is honored even when version='latest' is passed.""" + res = _get_databus_versions_of_artifact( + SAMPLE_ARTIFACT_JSONLD, all_versions=True, version="latest" + ) + assert isinstance(res, list) + assert len(res) == 3 + + +def test_get_versions_all_flag(): + """Verify all_versions=True returns list of all version URLs sorted newest-first.""" + res = _get_databus_versions_of_artifact(SAMPLE_ARTIFACT_JSONLD, all_versions=True) + assert isinstance(res, list) + assert len(res) == 3 + assert res[0] == "https://databus.dbpedia.org/user/group/my-artifact/2023.10.15" + assert res[-1] == "https://databus.dbpedia.org/user/group/my-artifact/2021.01.01" + + +def test_get_versions_version_all_string(): + """Verify version='ALL' (case-insensitive) returns all version URLs.""" + res = _get_databus_versions_of_artifact(SAMPLE_ARTIFACT_JSONLD, version="ALL") + assert isinstance(res, list) + assert len(res) == 3 + + +def test_get_versions_specific_version_id(): + """Verify specific version segment ID matches correct version URL.""" + res = _get_databus_versions_of_artifact(SAMPLE_ARTIFACT_JSONLD, version="2022.05.01") + assert res == "https://databus.dbpedia.org/user/group/my-artifact/2022.05.01" + + +def test_get_versions_specific_full_uri(): + """Verify exact matching full version URI matches correct version URL.""" + res = _get_databus_versions_of_artifact( + SAMPLE_ARTIFACT_JSONLD, + version="https://databus.dbpedia.org/user/group/my-artifact/2021.01.01", + ) + assert res == "https://databus.dbpedia.org/user/group/my-artifact/2021.01.01" + + +def test_get_versions_mismatched_full_uri_raises(): + """Verify full URI for a different artifact does not match despite identical version ID.""" + with pytest.raises( + ValueError, + match="Version 'https://databus.dbpedia.org/user/group/other-artifact/2021.01.01' not found", + ): + _get_databus_versions_of_artifact( + SAMPLE_ARTIFACT_JSONLD, + version="https://databus.dbpedia.org/user/group/other-artifact/2021.01.01", + ) + + +def test_get_versions_not_found_raises(): + """Verify non-existent version ID raises ValueError with available versions.""" + with pytest.raises(ValueError, match="Version '1999.01.01' not found"): + _get_databus_versions_of_artifact(SAMPLE_ARTIFACT_JSONLD, version="1999.01.01") + + +def test_get_versions_missing_field_raises(): + """Verify missing databus:hasVersion raises ValueError.""" + invalid_json = json.dumps({"@id": "https://databus.dbpedia.org/u/g/a"}) + with pytest.raises(ValueError, match="No 'databus:hasVersion' field"): + _get_databus_versions_of_artifact(invalid_json) + + +def test_get_versions_empty_list_raises(): + """Verify empty databus:hasVersion list raises ValueError.""" + invalid_json = json.dumps({"@id": "https://databus.dbpedia.org/u/g/a", "databus:hasVersion": []}) + with pytest.raises(ValueError, match="No versions found"): + _get_databus_versions_of_artifact(invalid_json) + + +# --------------------------------------------------------------------------- +# Integration tests: _download_artifact and _download_group +# --------------------------------------------------------------------------- + +def test_download_artifact_with_specific_version(monkeypatch): + """Verify _download_artifact downloads only the requested specific version.""" + fetched = [] + downloaded_files = [] + + def fake_fetch_jsonld(uri, databus_key=None): + fetched.append(uri) + if "2022.05.01" in uri: + return SAMPLE_VERSION_JSONLD + return SAMPLE_ARTIFACT_JSONLD + + def fake_download_files(urls, localDir, **kwargs): + downloaded_files.extend(urls) + + monkeypatch.setattr(dl, "fetch_databus_jsonld", fake_fetch_jsonld) + monkeypatch.setattr(dl, "_download_files", fake_download_files) + + dl._download_artifact( + "https://databus.dbpedia.org/user/group/my-artifact", + localDir="tmp", + version="2022.05.01", + ) + + assert "https://databus.dbpedia.org/user/group/my-artifact/2022.05.01" in fetched + assert "https://example.org/files/data_20220501.ttl" in downloaded_files + + +def test_download_group_with_specific_version_skips_unmatched_artifacts(monkeypatch, capsys): + """Verify _download_group skips artifacts not containing the requested version.""" + artifact_one_json = json.dumps({ + "@id": "https://databus.dbpedia.org/user/group/artifact-one", + "databus:hasVersion": [{"@id": "https://databus.dbpedia.org/user/group/artifact-one/2022.05.01"}], + }) + artifact_two_json = json.dumps({ + "@id": "https://databus.dbpedia.org/user/group/artifact-two", + "databus:hasVersion": [{"@id": "https://databus.dbpedia.org/user/group/artifact-two/2020.01.01"}], + }) + + def fake_fetch_jsonld(uri, databus_key=None): + if uri.endswith("/group"): + return SAMPLE_GROUP_JSONLD + if uri.endswith("artifact-one"): + return artifact_one_json + if uri.endswith("artifact-two"): + return artifact_two_json + if "2022.05.01" in uri: + return SAMPLE_VERSION_JSONLD + return "{}" + + downloaded = [] + monkeypatch.setattr(dl, "fetch_databus_jsonld", fake_fetch_jsonld) + monkeypatch.setattr(dl, "_download_files", lambda urls, *a, **k: downloaded.extend(urls)) + + dl._download_group( + "https://databus.dbpedia.org/user/group", + localDir="tmp", + version="2022.05.01", + ) + + captured = capsys.readouterr().out + assert "Skipping artifact https://databus.dbpedia.org/user/group/artifact-two" in captured + assert "https://example.org/files/data_20220501.ttl" in downloaded + + +def test_download_group_with_empty_version_artifact_skips_and_downloads_matching(monkeypatch, capsys): + """Verify _download_group skips artifacts with zero versions and continues download.""" + group_json = json.dumps({ + "@id": "https://databus.dbpedia.org/user/group", + "@type": "dataid:Group", + "databus:hasArtifact": [ + {"@id": "https://databus.dbpedia.org/user/group/artifact-empty"}, + {"@id": "https://databus.dbpedia.org/user/group/artifact-valid"}, + ], + }) + artifact_empty_json = json.dumps({ + "@id": "https://databus.dbpedia.org/user/group/artifact-empty", + "databus:hasVersion": [], + }) + artifact_valid_json = json.dumps({ + "@id": "https://databus.dbpedia.org/user/group/artifact-valid", + "databus:hasVersion": [{"@id": "https://databus.dbpedia.org/user/group/artifact-valid/2022.05.01"}], + }) + + def fake_fetch_jsonld(uri, databus_key=None): + if uri.endswith("/group"): + return group_json + if uri.endswith("artifact-empty"): + return artifact_empty_json + if uri.endswith("artifact-valid"): + return artifact_valid_json + if "2022.05.01" in uri: + return SAMPLE_VERSION_JSONLD + return "{}" + + downloaded = [] + monkeypatch.setattr(dl, "fetch_databus_jsonld", fake_fetch_jsonld) + monkeypatch.setattr(dl, "_download_files", lambda urls, *a, **k: downloaded.extend(urls)) + + dl._download_group( + "https://databus.dbpedia.org/user/group", + localDir="tmp", + version="2022.05.01", + ) + + captured = capsys.readouterr().out + assert "Skipping artifact https://databus.dbpedia.org/user/group/artifact-empty" in captured + assert "https://example.org/files/data_20220501.ttl" in downloaded + + +# --------------------------------------------------------------------------- +# API download dispatch tests +# --------------------------------------------------------------------------- + +def test_api_download_dispatches_version(monkeypatch): + """Verify api_download passes version parameter to artifact handler.""" + captured = {} + + def fake_download_artifact(uri, localDir, **kwargs): + captured["uri"] = uri + captured["version"] = kwargs.get("version") + captured["all_versions"] = kwargs.get("all_versions") + + monkeypatch.setattr(dl, "_download_artifact", fake_download_artifact) + + api_download( + localDir="tmp", + endpoint=None, + databusURIs=["https://databus.dbpedia.org/user/group/my-artifact"], + version="2022.05.01", + ) + + assert captured["version"] == "2022.05.01" + assert captured["all_versions"] is False + + +def test_api_download_all_versions_sets_version_all(monkeypatch): + """Verify all_versions=True sets version to 'all'.""" + captured = {} + + def fake_download_artifact(uri, localDir, **kwargs): + captured["all_versions"] = kwargs.get("all_versions") + captured["version"] = kwargs.get("version") + + monkeypatch.setattr(dl, "_download_artifact", fake_download_artifact) + + api_download( + localDir="tmp", + endpoint=None, + databusURIs=["https://databus.dbpedia.org/user/group/my-artifact"], + all_versions=True, + ) + + assert captured["all_versions"] is True + assert captured["version"] == "all" + + +def test_api_download_positional_arguments_preserve_auth_url(monkeypatch): + """Verify passing auth_url positionally binds to auth_url and not version.""" + captured = {} + + def fake_download_version(uri, localDir, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr(dl, "_download_version", fake_download_version) + + # Call with 7 positional arguments: localDir, endpoint, databusURIs, token, databus_key, all_versions, auth_url + api_download( + "tmp", + "https://databus.dbpedia.org/sparql", + ["https://databus.dbpedia.org/user/group/my-artifact/2022.05.01"], + "my-token.dat", + "my-key", + False, + "https://custom-auth.org/realms/token", + ) + + assert captured["auth_url"] == "https://custom-auth.org/realms/token" + assert captured.get("version") is None + + +# --------------------------------------------------------------------------- +# CLI tests: --version and --target-version +# --------------------------------------------------------------------------- + +def test_cli_download_version_flag(monkeypatch): + """Verify CLI --version flag forwards target version to api_download.""" + captured = {} + + def fake_api_download(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr("databusclient.cli.api_download", fake_api_download) + + runner = CliRunner() + result = runner.invoke( + app, + [ + "download", + "https://databus.dbpedia.org/user/group/my-artifact", + "--version", + "2022.05.01", + ], + ) + + assert result.exit_code == 0 + assert captured["version"] == "2022.05.01" + + +def test_cli_download_target_version_flag(monkeypatch): + """Verify CLI --target-version alias forwards target version to api_download.""" + captured = {} + + def fake_api_download(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr("databusclient.cli.api_download", fake_api_download) + + runner = CliRunner() + result = runner.invoke( + app, + [ + "download", + "https://databus.dbpedia.org/user/group/my-artifact", + "--target-version", + "2022.05.01", + ], + ) + + assert result.exit_code == 0 + assert captured["version"] == "2022.05.01" + + +def test_cli_download_records_version_in_manifest(monkeypatch, tmp_path): + """Verify CLI download records version in manifest replay parameters.""" + monkeypatch.setattr("databusclient.cli.api_download", lambda **k: None) + + manifest_file = tmp_path / "manifest.jsonld" + runner = CliRunner() + result = runner.invoke( + app, + [ + "download", + "https://databus.dbpedia.org/user/group/my-artifact", + "--version", + "2022.05.01", + "--manifest", + str(manifest_file), + ], + ) + + assert result.exit_code == 0 + data = json.loads(manifest_file.read_text(encoding="utf-8")) + assert data["dbus:replayParams"]["version"] == "2022.05.01" + + +# --------------------------------------------------------------------------- +# Manifest replay & Workflow step tests +# --------------------------------------------------------------------------- + +def test_manifest_replay_restores_version(monkeypatch, tmp_path): + """Verify manifest replay extracts and passes version to api_download.""" + captured = {} + + def fake_api_download(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr("databusclient.manifest.replay.api_download", fake_api_download) + + manifest = { + "@type": "dbus:OperationManifest", + "dbus:command": "download", + "dbus:endpoint": "https://databus.dbpedia.org/sparql", + "dbus:replayParams": { + "databusURIs": ["https://databus.dbpedia.org/user/group/my-artifact"], + "version": "2022.05.01", + }, + } + manifest_path = tmp_path / "replay_version.jsonld" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + replay_manifest(str(manifest_path)) + + assert captured["version"] == "2022.05.01" + + +def test_manifest_replay_rejects_non_string_version(monkeypatch, tmp_path): + """Verify manifest replay raises ManifestReplayError for non-string version.""" + monkeypatch.setattr("databusclient.manifest.replay.api_download", lambda **k: None) + + manifest = { + "@type": "dbus:OperationManifest", + "dbus:command": "download", + "dbus:endpoint": "https://databus.dbpedia.org/sparql", + "dbus:replayParams": { + "databusURIs": ["https://databus.dbpedia.org/user/group/my-artifact"], + "version": 20220501, # Non-string integer + }, + } + manifest_path = tmp_path / "replay_invalid_version.jsonld" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises( + ManifestReplayError, + match="Manifest field dbus:replayParams.version must be a string.", + ): + replay_manifest(str(manifest_path)) + + +def test_workflow_step_executes_version(monkeypatch, tmp_path): + """Verify declarative workflow DownloadStep passes version to api_download.""" + captured = {} + + def fake_api_download(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr("databusclient.workflow.steps.api_download", fake_api_download) + + step = { + "name": "fetch_version", + "action": "download", + "uris": ["https://databus.dbpedia.org/user/group/my-artifact"], + "version": "2022.05.01", + "localdir": str(tmp_path), + } + context = StepContext() + DownloadStep().run(step, context) + + assert captured["version"] == "2022.05.01"