diff --git a/databusclient/api/download.py b/databusclient/api/download.py index 356a2dd..5b8abfc 100644 --- a/databusclient/api/download.py +++ b/databusclient/api/download.py @@ -7,7 +7,7 @@ import re import shutil import tempfile -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse import requests from SPARQLWrapper import JSON, SPARQLWrapper @@ -467,7 +467,7 @@ def _download_file( 307, 308, ]: - url = response.headers.get("Location") + url = urljoin(url, response.headers["Location"]) print("Redirects url: ", url) # Re-do HEAD request on redirect URL response = requests.head(url, timeout=30) diff --git a/tests/test_download_redirects.py b/tests/test_download_redirects.py new file mode 100644 index 0000000..8a5b456 --- /dev/null +++ b/tests/test_download_redirects.py @@ -0,0 +1,57 @@ +"""Exercise download redirects against a local HTTP server.""" + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread + +import pytest + +from databusclient.api.download import _download_file + + +@pytest.mark.parametrize( + ("location", "expected_path"), + [ + ("/data.ttl", "/data.ttl"), + ("../data.ttl", "/data.ttl"), + ("data.ttl", "/entry/data.ttl"), + ("{base}/data.ttl", "/data.ttl"), + ("//{host}/data.ttl", "/data.ttl"), + ], +) +def test_download_resolves_redirect_location(tmp_path, location, expected_path): + content = b" .\n" + requested_paths = [] + + class Handler(BaseHTTPRequestHandler): + def do_HEAD(self): + if self.path == "/entry/file.ttl": + self.send_response(302) + self.send_header("Location", redirect_location) + else: + self.send_response(200) + self.end_headers() + + def do_GET(self): + requested_paths.append(self.path) + self.send_response(200) + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + + def log_message(self, format, *args): + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + host = f"127.0.0.1:{server.server_port}" + base = f"http://{host}" + redirect_location = location.format(base=base, host=host) + thread = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}) + thread.start() + try: + _download_file(f"{base}/entry/file.ttl", localDir=str(tmp_path)) + finally: + server.shutdown() + thread.join() + + assert requested_paths == [expected_path] + assert (tmp_path / "file.ttl").read_bytes() == content