diff --git a/docs/serverless/volume_cache.md b/docs/serverless/volume_cache.md index 79d9e6e3a..4351293a1 100644 --- a/docs/serverless/volume_cache.md +++ b/docs/serverless/volume_cache.md @@ -60,8 +60,8 @@ vc.sync(background=False) # persist new files back to the volume, inli | `volume_path` | `/runpod-volume` | Network-volume mount point. | | `best_effort` | `True` | Swallow and log errors instead of raising. Set `False` while debugging. | | `max_workers` | `min(32, (os.cpu_count() or 4) * 4)` | Thread count for parallel copy of large files on sync (I/O-bound). | -| `hydrate_workers` | `max_workers` if set, else `64` | Concurrent range reads when hydrating large files. | -| `hydrate_chunk_size` | `8 MiB` | Bytes per hydrate range. Peak hydrate memory is about `hydrate_workers * hydrate_chunk_size` (512 MiB by default). | +| `hydrate_workers` | `max_workers` if set, else `8` | Concurrent range reads when hydrating large files. | +| `hydrate_chunk_size` | `512 MiB / hydrate_workers`, at most `64 MiB` | Bytes per hydrate range. Peak hydrate memory is about `hydrate_workers * hydrate_chunk_size`, so the default keeps it at or below 512 MiB. | ## How it works @@ -97,7 +97,11 @@ vc.sync(background=False) # persist new files back to the volume, inli likewise not restored. This is a best-effort check: an object-store mount can serve a replacement that lands in the middle of a single range read without it being visible. Keep `hydrate_workers * hydrate_chunk_size` near the default: past ~1 GiB in - flight, Global Store starts failing reads with `ENOMEM`. + flight, Global Store starts failing reads with `ENOMEM`. The default of 8 x + 64 MiB ranges is about 2x faster than 64 x 8 MiB on network volumes and + within run-to-run noise on Global Store; if first-hydrate times on fresh + Global Store workers matter, `hydrate_workers=64` (8 MiB ranges) is worth + measuring. - **Packing/extraction.** Packing the small-file archive uses the `tar` binary when available (falling back to the stdlib `tarfile` module otherwise). Extraction always goes through `tarfile`, validating every diff --git a/runpod/serverless/utils/rp_volume_cache.py b/runpod/serverless/utils/rp_volume_cache.py index 344ac9d41..354f67dca 100644 --- a/runpod/serverless/utils/rp_volume_cache.py +++ b/runpod/serverless/utils/rp_volume_cache.py @@ -46,11 +46,15 @@ _MANIFEST_NAME = "manifest.json" _SMALL_ARCHIVE_NAME = "small.tar" _BIG_SUBDIR = "big" -# Hydrate copies big files in ranges. Bytes in flight (workers x chunk, 512 MiB -# here) is what matters: on Global Store's geesefs mount, 64 x 8 MiB reaches the -# host NIC (~1.1 GiB/s), and much past ~1 GiB in flight fails reads with ENOMEM. -_HYDRATE_WORKERS = 64 -_HYDRATE_CHUNK_SIZE = 8 * 1024 * 1024 +# Hydrate copies big files in ranges. Bytes in flight (workers x chunk) is what +# matters: Global Store's geesefs mount fails reads with ENOMEM much past ~1 GiB +# in flight, so the default chunk is sized to keep the pool within the budget. +# At 512 MiB, 8 x 64 MiB measured 2-2.8x faster than 64 x 8 MiB on a network +# volume (7B: ~1420 vs ~730 MiB/s; 3B: ~1000 vs ~360) and within run-to-run +# noise on Global Store, with no ENOMEM even on a busy 8-GPU host. +_HYDRATE_WORKERS = 8 +_HYDRATE_BYTES_IN_FLIGHT = 512 * 1024 * 1024 +_HYDRATE_MAX_CHUNK_SIZE = 64 * 1024 * 1024 # Transient errors an object-store FUSE mount returns under load; retried with # jittered exponential backoff (up to ~6s in total) before the file is given up # on. Jitter keeps ranges that failed together from retrying in lockstep. @@ -86,9 +90,11 @@ class VolumeCache: Defaults to ``min(32, (os.cpu_count() or 4) * 4)`` (I/O-bound, so oversubscribing the CPU count is fine). hydrate_workers: Concurrent range reads on hydrate. Defaults to - ``max_workers`` when that is set, else 64. - hydrate_chunk_size: Bytes per hydrate range. Defaults to 8 MiB. Peak - hydrate memory is about ``hydrate_workers * hydrate_chunk_size``. + ``max_workers`` when that is set, else 8. + hydrate_chunk_size: Bytes per hydrate range. Defaults to 512 MiB + divided by ``hydrate_workers``, capped at 64 MiB (64 MiB with the + default 8 workers). Peak hydrate memory is about + ``hydrate_workers * hydrate_chunk_size``. Example: >>> with VolumeCache(dirs=["/root/.cache/huggingface"]): @@ -142,7 +148,7 @@ def __init__( "hydrate_workers", hydrate_workers, max_workers or _HYDRATE_WORKERS ) self._hydrate_chunk_size = _positive_or_default( - "hydrate_chunk_size", hydrate_chunk_size, _HYDRATE_CHUNK_SIZE + "hydrate_chunk_size", hydrate_chunk_size, _default_chunk_size(self._hydrate_workers) ) @property @@ -671,6 +677,11 @@ def _copy_metadata(src, dst, mtime=None): log.debug(f"VolumeCache: kept default mode on {dst}: {exc}") +def _default_chunk_size(workers): + # Larger pools get smaller ranges so bytes in flight stay within budget. + return max(1, min(_HYDRATE_MAX_CHUNK_SIZE, _HYDRATE_BYTES_IN_FLIGHT // workers)) + + def _positive_or_default(name, value, default): # A non-positive or non-integer pool size or chunk is a caller bug that # best-effort mode would otherwise swallow mid-operation (range() and diff --git a/tests/test_serverless/test_utils/test_rp_volume_cache.py b/tests/test_serverless/test_utils/test_rp_volume_cache.py index b49d77df7..d0ba3e03a 100644 --- a/tests/test_serverless/test_utils/test_rp_volume_cache.py +++ b/tests/test_serverless/test_utils/test_rp_volume_cache.py @@ -12,6 +12,7 @@ import runpod.serverless.utils.rp_volume_cache as vcmod VolumeCache = vcmod.VolumeCache +MIB = 1024 * 1024 @pytest.fixture(autouse=True) @@ -1345,9 +1346,48 @@ def submit(self, *args, **kwargs): def test_hydrate_tuning_defaults(tmp_path): + # 8 x 64 MiB: ~2.8x faster than 64 x 8 MiB on network volumes and no slower + # on Global Store, at the same 512 MiB in flight. vc, _cache, _vol = _mk_cache_with_volume(tmp_path) - assert vc._hydrate_workers == 64 - assert vc._hydrate_chunk_size == 8 * 1024 * 1024 + assert vc._hydrate_workers == 8 + assert vc._hydrate_chunk_size == 64 * MIB + + +@pytest.mark.parametrize( + ("workers", "chunk"), + [(4, 64 * MIB), (16, 32 * MIB), (32, 16 * MIB), (64, 8 * MIB)], +) +def test_default_hydrate_chunk_keeps_bytes_in_flight_bounded(tmp_path, workers, chunk): + # geesefs fails reads with ENOMEM much past ~1 GiB in flight, so a larger + # pool must get smaller default ranges, capped at 64 MiB for small pools. + vol = tmp_path / "volume" + vol.mkdir() + vc = VolumeCache( + [str(tmp_path / "cache")], namespace="ep1", volume_path=str(vol), hydrate_workers=workers + ) + assert vc._hydrate_chunk_size == chunk + assert vc._hydrate_workers * vc._hydrate_chunk_size <= 512 * MIB + + +def test_default_hydrate_chunk_follows_explicit_max_workers(tmp_path): + vol = tmp_path / "volume" + vol.mkdir() + vc = VolumeCache([str(tmp_path / "cache")], namespace="ep1", volume_path=str(vol), max_workers=32) + assert vc._hydrate_workers == 32 + assert vc._hydrate_chunk_size == 16 * MIB + + +def test_explicit_hydrate_chunk_size_is_not_rescaled(tmp_path): + vol = tmp_path / "volume" + vol.mkdir() + vc = VolumeCache( + [str(tmp_path / "cache")], + namespace="ep1", + volume_path=str(vol), + hydrate_workers=32, + hydrate_chunk_size=32 * MIB, + ) + assert vc._hydrate_chunk_size == 32 * MIB def test_hydrate_skips_file_when_local_temp_cannot_be_created(tmp_path, monkeypatch):