From a8a8f73e979f8837ec5e0ca53c5039de39621e00 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Fri, 6 Nov 2026 22:56:19 +0100 Subject: [PATCH] GPU Metal: allocate the memory pools from anonymous memory The full system test publishes its time frames seconds apart, and after a few idle seconds the first command buffer of a time frame waits until the driver has made the memory pools resident again: in a Metal System Trace its driver processing takes about 165 ms, with "Wire 5.59 GiB System Memory" for the 6 GB device pool. That is the ~160 ms by which a time frame in the full system test was slower than the same time frame in the standalone benchmark, which never idles; with 2 s between the time frames the two take the same time. The driver does this faster for a buffer around anonymous memory than for one Metal allocates: 77 ms instead of 125 ms for 6 GB after 8 s idle on an M1 Max, in a standalone test. So wrap mmap'ed memory with newBufferWithBytesNoCopy for the device and host pools. On an M1 Max, interleaved over 4 rounds, the warm time frames of the TPC-only reconstruction of the 10-collision Pb-Pb full system test take 955 ms instead of 1003 (median of the chain's wall time, 8 s between the time frames), with the same 1,355,432 clusters. The 30-collision Pb-Pb benchmark is unchanged (970 against 967 ms per event over 3 rounds), and its clusters are byte-identical to the CPU's, as on sim5pp. --- .../Base/metal/GPUReconstructionMetal.mm | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm index 4be4c83ebc262..1a23b201da209 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm @@ -23,6 +23,8 @@ #include #include #include // _mh_execute_header +#include +#include #define GPUErrorReturn(...) \ { \ @@ -50,6 +52,30 @@ uint32_t metalGPUCoreCount() } return n; } + +// A shared buffer around anonymous memory rather than one Metal allocates. After +// the GPU has been idle for a few seconds, the first command buffer waits until +// the driver has made the memory pools resident again, which for 6 GB takes about +// 75 ms instead of 125 ms on an M1 Max, and the full system test idles between +// all its time frames. +id metalNewSharedBuffer(id device, size_t size) +{ + const size_t length = (size + getpagesize() - 1) & ~(size_t)(getpagesize() - 1); + void* p = mmap(nullptr, length, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + if (p == MAP_FAILED) { + return nil; + } + id buffer = [device newBufferWithBytesNoCopy:p + length:length + options:MTLResourceStorageModeShared | MTLResourceHazardTrackingModeUntracked + deallocator:^(void* ptr, NSUInteger l) { + munmap(ptr, l); + }]; + if (buffer == nil) { + munmap(p, length); + } + return buffer; +} } // namespace #include "utils/qGetLdBinarySymbols.h" @@ -116,7 +142,7 @@ uint32_t metalGPUCoreCount() } // We only support internal GPUs, so the ownership of the memory is always shared - mInternals->mem_gpu = [mInternals->device newBufferWithLength:mDeviceMemorySize options:MTLResourceStorageModeShared | MTLResourceHazardTrackingModeUntracked]; + mInternals->mem_gpu = metalNewSharedBuffer(mInternals->device, mDeviceMemorySize); if (mInternals->mem_gpu == nil) { GPUErrorReturn("Metal Memory Allocation Error"); } @@ -143,7 +169,7 @@ uint32_t metalGPUCoreCount() mInternals->streamHasWork[i] = false; } - mInternals->mem_host = [mInternals->device newBufferWithLength:mHostMemorySize options:MTLResourceStorageModeShared | MTLResourceHazardTrackingModeUntracked]; + mInternals->mem_host = metalNewSharedBuffer(mInternals->device, mHostMemorySize); if (mInternals->mem_host == nil) { GPUErrorReturn("Error allocating pinned host memory"); }