diff --git a/README.md b/README.md index e6ed0b7..65de76b 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ The command-line interface provides commands for downloading, deploying, and del ### Download -The `download` command retrieves Databus files, versions, artifacts, groups, collections, or SPARQL query results. It supports authentication, checksum validation, compression conversion, and RDF or tabular format conversion. See the [download docs](doc/cli-usage.md#cli-download). +The `download` command retrieves Databus files, versions, artifacts, groups, accounts, collections, or SPARQL query results. It supports authentication, checksum validation, compression conversion, and RDF or tabular format conversion. See the [download docs](doc/cli-usage.md#cli-download). ### Deploy diff --git a/databusclient/api/download.py b/databusclient/api/download.py index 356a2dd..b423e00 100644 --- a/databusclient/api/download.py +++ b/databusclient/api/download.py @@ -1374,6 +1374,122 @@ def _get_databus_artifacts_of_group(json_str: str) -> List[str]: return result +def _get_groups_of_account( + account_uri: str, + endpoint_url: str, + databus_key: str | None = None, +) -> List[str]: + """Query a Databus SPARQL endpoint to find all groups belonging to an account. + + Args: + account_uri: The full databus account URI (e.g. 'https://databus.dbpedia.org/dbpedia'). + endpoint_url: The Databus SPARQL endpoint URL. + databus_key: Optional Databus API key for authentication on protected databus. + + Returns: + List of group URIs belonging to the account. + """ + clean_account = account_uri.rstrip("/") + query_v2 = f"""PREFIX databus: +SELECT DISTINCT ?group WHERE {{ + ?dataset databus:account <{clean_account}> ; + databus:group ?group . +}} ORDER BY ?group""" + + try: + result = _query_sparql_endpoint(endpoint_url, query_v2, databus_key=databus_key) + bindings = result.get("results", {}).get("bindings", []) + except Exception as e: + print(f"Warning: SPARQL query for account groups failed: {e}") + bindings = [] + + if not bindings: + query_legacy = f"""PREFIX dataid: +SELECT DISTINCT ?group WHERE {{ + ?dataset dataid:account <{clean_account}> ; + dataid:group ?group . +}} ORDER BY ?group""" + try: + result = _query_sparql_endpoint(endpoint_url, query_legacy, databus_key=databus_key) + bindings = result.get("results", {}).get("bindings", []) + except Exception: + bindings = [] + + group_uris: List[str] = [] + for binding in bindings: + group_val = binding.get("group", {}).get("value") + if isinstance(group_val, str) and group_val.startswith(f"{clean_account}/"): + if group_val not in group_uris: + group_uris.append(group_val) + + return group_uris + + +def _download_account( + uri: str, + endpoint: str, + localDir: str, + all_versions: bool = False, + vault_token_file: str = None, + databus_key: str = None, + auth_url: str = None, + client_id: str = None, + compression: str = None, + convert_format: str = None, + graph_name: str = None, + base_uri: str = None, + graph_mode: str = None, + manifest_context=None, + validate_checksum: bool = False, +) -> None: + """Download all groups, artifacts and versions of an account from Databus. + + Args: + uri: The full databus account URI. + endpoint: The databus SPARQL endpoint URL. + localDir: Local directory to download files to. If None, the databus folder structure is created in the current working directory. + all_versions: If True, download all versions of each artifact in the account; otherwise, only download the latest version. + vault_token_file: Path to Vault refresh token file for protected downloads. + databus_key: Databus API key for protected downloads. + auth_url: Keycloak token endpoint URL. + client_id: Client ID for token exchange. + compression: Target compression format for on-the-fly conversion. + convert_format: Target RDF/tabular format for on-the-fly conversion. + graph_name: Named graph URI for Triple -> Quad conversion (Layer 3). + base_uri: Base URI for CSV -> Triple conversion (Layer 3). + graph_mode: Optional graph sidecar mode. Currently supports 'download-url'. + validate_checksum: Whether to validate checksums after downloading. + """ + clean_account = uri.rstrip("/") + group_uris = _get_groups_of_account(clean_account, endpoint, databus_key=databus_key) + if not group_uris: + print(f"No groups found for account: {clean_account}") + return + + print(f"Found {len(group_uris)} group(s) for account {clean_account}:") + for g in group_uris: + print(f" - {g}") + + for group_uri in group_uris: + print(f"Downloading group: {group_uri}") + _download_group( + group_uri, + localDir, + all_versions=all_versions, + vault_token_file=vault_token_file, + databus_key=databus_key, + auth_url=auth_url, + client_id=client_id, + compression=compression, + convert_format=convert_format, + graph_name=graph_name, + base_uri=base_uri, + graph_mode=graph_mode, + manifest_context=manifest_context, + validate_checksum=validate_checksum, + ) + + def download( localDir: str, endpoint: str, @@ -1393,7 +1509,7 @@ def download( ) -> None: """Download datasets from databus. - Download of files, versions, artifacts, groups or databus collections via their databus URIs or user-defined SPARQL queries that return file download URLs. + Download of files, versions, artifacts, groups, accounts or databus collections via their databus URIs or user-defined SPARQL queries that return file download URLs. Args: localDir: Local directory to download datasets to. If None, the databus folder structure is created in the current working directory. @@ -1527,7 +1643,24 @@ def download( validate_checksum=validate_checksum, ) elif account is not None: - print("accountId not supported yet") # TODO + print(f"Downloading account and all its groups: {databusURI}") + _download_account( + databusURI, + uri_endpoint, + localDir, + all_versions=all_versions, + vault_token_file=token, + databus_key=databus_key, + auth_url=auth_url, + client_id=client_id, + compression=compression, + convert_format=convert_format, + graph_name=graph_name, + base_uri=base_uri, + graph_mode=graph_mode, + manifest_context=manifest_context, + validate_checksum=validate_checksum, + ) else: print( "dataId not supported yet" diff --git a/doc/cli-usage.md b/doc/cli-usage.md index 1325220..ab68ba3 100644 --- a/doc/cli-usage.md +++ b/doc/cli-usage.md @@ -12,7 +12,7 @@ databusclient [delete|deploy|download|manifest|workflow] --help ### Download -With the download command, you can download datasets or parts thereof from the Databus. The download command expects one or more Databus URIs or a SPARQL query as arguments. The URIs can point to files, versions, artifacts, groups, or collections. If a SPARQL query is provided, the query must return download URLs from the Databus which will be downloaded. +With the download command, you can download datasets or parts thereof from the Databus. The download command expects one or more Databus URIs or a SPARQL query as arguments. The URIs can point to files, versions, artifacts, groups, accounts, or collections. If a SPARQL query is provided, the query must return download URLs from the Databus which will be downloaded. ```bash databusclient download $DOWNLOADTARGET @@ -63,6 +63,11 @@ databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals ``` +**Download Account**: download of all files with the latest version of all artifacts across all groups owned by an account +```bash +databusclient download https://databus.dbpedia.org/dbpedia +``` + **Download Group**: download of all files with the latest version of all artifacts of a group ```bash databusclient download https://databus.dbpedia.org/dbpedia/mappings diff --git a/tests/test_download_account.py b/tests/test_download_account.py new file mode 100644 index 0000000..8b8a95b --- /dev/null +++ b/tests/test_download_account.py @@ -0,0 +1,178 @@ +"""Tests for downloading all datasets of an account from Databus.""" + +from unittest.mock import MagicMock +from click.testing import CliRunner + +from databusclient.api.download import ( + _get_groups_of_account, + _download_account, + download as api_download, +) +from databusclient.cli import app + + +def test_get_groups_of_account_v2(monkeypatch): + fake_results = { + "results": { + "bindings": [ + {"group": {"value": "https://databus.dbpedia.org/myuser/group1"}}, + {"group": {"value": "https://databus.dbpedia.org/myuser/group2"}}, + {"group": {"value": "https://databus.dbpedia.org/otheruser/groupX"}}, # should filter out + ] + } + } + + monkeypatch.setattr( + "databusclient.api.download._query_sparql_endpoint", + lambda endpoint, query, databus_key=None: fake_results, + ) + + groups = _get_groups_of_account( + "https://databus.dbpedia.org/myuser", + "https://databus.dbpedia.org/sparql", + ) + assert groups == [ + "https://databus.dbpedia.org/myuser/group1", + "https://databus.dbpedia.org/myuser/group2", + ] + + +def test_get_groups_of_account_legacy_fallback(monkeypatch): + call_count = 0 + + def fake_query(endpoint, query, databus_key=None): + nonlocal call_count + call_count += 1 + if call_count == 1: + # v2 query returns empty + return {"results": {"bindings": []}} + else: + # legacy query returns result + return { + "results": { + "bindings": [ + {"group": {"value": "https://databus.dbpedia.org/legacyuser/group1"}}, + ] + } + } + + monkeypatch.setattr("databusclient.api.download._query_sparql_endpoint", fake_query) + + groups = _get_groups_of_account( + "https://databus.dbpedia.org/legacyuser", + "https://databus.dbpedia.org/sparql", + ) + assert call_count == 2 + assert groups == ["https://databus.dbpedia.org/legacyuser/group1"] + + +def test_get_groups_of_account_empty_and_error_handling(monkeypatch): + def fake_query_err(endpoint, query, databus_key=None): + raise RuntimeError("SPARQL error") + + monkeypatch.setattr("databusclient.api.download._query_sparql_endpoint", fake_query_err) + + groups = _get_groups_of_account( + "https://databus.dbpedia.org/brokenuser", + "https://databus.dbpedia.org/sparql", + ) + assert groups == [] + + +def test_download_account_downloads_each_group(monkeypatch): + mock_download_group = MagicMock() + monkeypatch.setattr("databusclient.api.download._download_group", mock_download_group) + monkeypatch.setattr( + "databusclient.api.download._get_groups_of_account", + lambda account, endpoint, databus_key=None: [ + "https://databus.dbpedia.org/testuser/group1", + "https://databus.dbpedia.org/testuser/group2", + ], + ) + + _download_account( + "https://databus.dbpedia.org/testuser", + endpoint="https://databus.dbpedia.org/sparql", + localDir="/tmp/test", + all_versions=True, + ) + + assert mock_download_group.call_count == 2 + mock_download_group.assert_any_call( + "https://databus.dbpedia.org/testuser/group1", + "/tmp/test", + all_versions=True, + vault_token_file=None, + databus_key=None, + auth_url=None, + client_id=None, + compression=None, + convert_format=None, + graph_name=None, + base_uri=None, + graph_mode=None, + manifest_context=None, + validate_checksum=False, + ) + + +def test_download_account_no_groups(monkeypatch): + mock_download_group = MagicMock() + monkeypatch.setattr("databusclient.api.download._download_group", mock_download_group) + monkeypatch.setattr( + "databusclient.api.download._get_groups_of_account", + lambda account, endpoint, databus_key=None: [], + ) + + _download_account( + "https://databus.dbpedia.org/emptyuser", + endpoint="https://databus.dbpedia.org/sparql", + localDir="/tmp/test", + ) + + mock_download_group.assert_not_called() + + +def test_download_dispatch_detects_account(monkeypatch): + mock_download_account = MagicMock() + monkeypatch.setattr("databusclient.api.download._download_account", mock_download_account) + + api_download( + localDir="/tmp/out", + endpoint="https://databus.dbpedia.org/sparql", + databusURIs=["https://databus.dbpedia.org/myorg"], + all_versions=False, + ) + + mock_download_account.assert_called_once_with( + "https://databus.dbpedia.org/myorg", + "https://databus.dbpedia.org/sparql", + "/tmp/out", + all_versions=False, + vault_token_file=None, + databus_key=None, + auth_url="https://auth.dbpedia.org/realms/dbpedia/protocol/openid-connect/token", + client_id="vault-token-exchange", + compression=None, + convert_format=None, + graph_name=None, + base_uri=None, + graph_mode=None, + manifest_context=None, + validate_checksum=False, + ) + + +def test_cli_download_account(monkeypatch): + mock_download = MagicMock() + monkeypatch.setattr("databusclient.cli.api_download", mock_download) + + runner = CliRunner() + result = runner.invoke( + app, + ["download", "https://databus.dbpedia.org/myuser", "--localdir", "/data"], + ) + + assert result.exit_code == 0 + mock_download.assert_called_once() + assert "https://databus.dbpedia.org/myuser" in mock_download.call_args[1]["databusURIs"]