perf(serverless): 8x64 MiB VolumeCache hydrate

docs/serverless/volume_cache.md

@@ -60,8 +60,8 @@ vc.sync(background=False)           # persist new files back to the volume, inli
 | `volume_path` | `/runpod-volume` | Network-volume mount point. |
 | `best_effort` | `True` | Swallow and log errors instead of raising. Set `False` while debugging. |
 | `max_workers` | `min(32, (os.cpu_count() or 4) * 4)` | Thread count for parallel copy of large files on sync (I/O-bound). |
-| `hydrate_workers` | `max_workers` if set, else `64` | Concurrent range reads when hydrating large files. |
-| `hydrate_chunk_size` | `8 MiB` | Bytes per hydrate range. Peak hydrate memory is about `hydrate_workers * hydrate_chunk_size` (512 MiB by default). |
+| `hydrate_workers` | `max_workers` if set, else `8` | Concurrent range reads when hydrating large files. |
+| `hydrate_chunk_size` | `512 MiB / hydrate_workers`, at most `64 MiB` | Bytes per hydrate range. Peak hydrate memory is about `hydrate_workers * hydrate_chunk_size`, so the default keeps it at or below 512 MiB. |
 
 ## How it works
 
@@ -97,7 +97,11 @@ vc.sync(background=False)           # persist new files back to the volume, inli
   likewise not restored. This is a best-effort check: an object-store mount
   can serve a replacement that lands in the middle of a single range read
   without it being visible. Keep `hydrate_workers * hydrate_chunk_size` near the default: past ~1 GiB in
-  flight, Global Store starts failing reads with `ENOMEM`.
+  flight, Global Store starts failing reads with `ENOMEM`. The default of 8 x
+  64 MiB ranges is about 2x faster than 64 x 8 MiB on network volumes and
+  within run-to-run noise on Global Store; if first-hydrate times on fresh
+  Global Store workers matter, `hydrate_workers=64` (8 MiB ranges) is worth
+  measuring.
 - **Packing/extraction.** Packing the small-file archive uses the `tar`
   binary when available (falling back to the stdlib `tarfile` module
   otherwise). Extraction always goes through `tarfile`, validating every

runpod/serverless/utils/rp_volume_cache.py

@@ -46,11 +46,15 @@
 _MANIFEST_NAME = "manifest.json"
 _SMALL_ARCHIVE_NAME = "small.tar"
 _BIG_SUBDIR = "big"
-# Hydrate copies big files in ranges. Bytes in flight (workers x chunk, 512 MiB
-# here) is what matters: on Global Store's geesefs mount, 64 x 8 MiB reaches the
-# host NIC (~1.1 GiB/s), and much past ~1 GiB in flight fails reads with ENOMEM.
-_HYDRATE_WORKERS = 64
-_HYDRATE_CHUNK_SIZE = 8 * 1024 * 1024
+# Hydrate copies big files in ranges. Bytes in flight (workers x chunk) is what
+# matters: Global Store's geesefs mount fails reads with ENOMEM much past ~1 GiB
+# in flight, so the default chunk is sized to keep the pool within the budget.
+# At 512 MiB, 8 x 64 MiB measured 2-2.8x faster than 64 x 8 MiB on a network
+# volume (7B: ~1420 vs ~730 MiB/s; 3B: ~1000 vs ~360) and within run-to-run
+# noise on Global Store, with no ENOMEM even on a busy 8-GPU host.
+_HYDRATE_WORKERS = 8
+_HYDRATE_BYTES_IN_FLIGHT = 512 * 1024 * 1024
+_HYDRATE_MAX_CHUNK_SIZE = 64 * 1024 * 1024
 # Transient errors an object-store FUSE mount returns under load; retried with
 # jittered exponential backoff (up to ~6s in total) before the file is given up
 # on. Jitter keeps ranges that failed together from retrying in lockstep.
@@ -86,9 +90,11 @@ class VolumeCache:
             Defaults to ``min(32, (os.cpu_count() or 4) * 4)`` (I/O-bound, so
             oversubscribing the CPU count is fine).
         hydrate_workers: Concurrent range reads on hydrate. Defaults to
-            ``max_workers`` when that is set, else 64.
-        hydrate_chunk_size: Bytes per hydrate range. Defaults to 8 MiB. Peak
-            hydrate memory is about ``hydrate_workers * hydrate_chunk_size``.
+            ``max_workers`` when that is set, else 8.
+        hydrate_chunk_size: Bytes per hydrate range. Defaults to 512 MiB
+            divided by ``hydrate_workers``, capped at 64 MiB (64 MiB with the
+            default 8 workers). Peak hydrate memory is about
+            ``hydrate_workers * hydrate_chunk_size``.
 
     Example:
         >>> with VolumeCache(dirs=["/root/.cache/huggingface"]):
@@ -142,7 +148,7 @@ def __init__(
             "hydrate_workers", hydrate_workers, max_workers or _HYDRATE_WORKERS
         )
         self._hydrate_chunk_size = _positive_or_default(
-            "hydrate_chunk_size", hydrate_chunk_size, _HYDRATE_CHUNK_SIZE
+            "hydrate_chunk_size", hydrate_chunk_size, _default_chunk_size(self._hydrate_workers)
         )
 
     @property
@@ -671,6 +677,11 @@ def _copy_metadata(src, dst, mtime=None):
         log.debug(f"VolumeCache: kept default mode on {dst}: {exc}")
 
 
+def _default_chunk_size(workers):
+    # Larger pools get smaller ranges so bytes in flight stay within budget.
+    return max(1, min(_HYDRATE_MAX_CHUNK_SIZE, _HYDRATE_BYTES_IN_FLIGHT // workers))
+
+
 def _positive_or_default(name, value, default):
     # A non-positive or non-integer pool size or chunk is a caller bug that
     # best-effort mode would otherwise swallow mid-operation (range() and

tests/test_serverless/test_utils/test_rp_volume_cache.py

@@ -12,6 +12,7 @@
 import runpod.serverless.utils.rp_volume_cache as vcmod
 
 VolumeCache = vcmod.VolumeCache
+MIB = 1024 * 1024
 
 
 @pytest.fixture(autouse=True)
@@ -1345,9 +1346,48 @@ def submit(self, *args, **kwargs):
 
 
 def test_hydrate_tuning_defaults(tmp_path):
+    # 8 x 64 MiB: ~2.8x faster than 64 x 8 MiB on network volumes and no slower
+    # on Global Store, at the same 512 MiB in flight.
     vc, _cache, _vol = _mk_cache_with_volume(tmp_path)
-    assert vc._hydrate_workers == 64
-    assert vc._hydrate_chunk_size == 8 * 1024 * 1024
+    assert vc._hydrate_workers == 8
+    assert vc._hydrate_chunk_size == 64 * MIB
+
+
+@pytest.mark.parametrize(
+    ("workers", "chunk"),
+    [(4, 64 * MIB), (16, 32 * MIB), (32, 16 * MIB), (64, 8 * MIB)],
+)
+def test_default_hydrate_chunk_keeps_bytes_in_flight_bounded(tmp_path, workers, chunk):
+    # geesefs fails reads with ENOMEM much past ~1 GiB in flight, so a larger
+    # pool must get smaller default ranges, capped at 64 MiB for small pools.
+    vol = tmp_path / "volume"
+    vol.mkdir()
+    vc = VolumeCache(
+        [str(tmp_path / "cache")], namespace="ep1", volume_path=str(vol), hydrate_workers=workers
+    )
+    assert vc._hydrate_chunk_size == chunk
+    assert vc._hydrate_workers * vc._hydrate_chunk_size <= 512 * MIB
+
+
+def test_default_hydrate_chunk_follows_explicit_max_workers(tmp_path):
+    vol = tmp_path / "volume"
+    vol.mkdir()
+    vc = VolumeCache([str(tmp_path / "cache")], namespace="ep1", volume_path=str(vol), max_workers=32)
+    assert vc._hydrate_workers == 32
+    assert vc._hydrate_chunk_size == 16 * MIB
+
+
+def test_explicit_hydrate_chunk_size_is_not_rescaled(tmp_path):
+    vol = tmp_path / "volume"
+    vol.mkdir()
+    vc = VolumeCache(
+        [str(tmp_path / "cache")],
+        namespace="ep1",
+        volume_path=str(vol),
+        hydrate_workers=32,
+        hydrate_chunk_size=32 * MIB,
+    )
+    assert vc._hydrate_chunk_size == 32 * MIB
 
 
 def test_hydrate_skips_file_when_local_temp_cannot_be_created(tmp_path, monkeypatch):