From 40a007ba4e26274f9943a2437d29abb64153cb3b Mon Sep 17 00:00:00 2001 From: Gerard Gorman Date: Mon, 26 Oct 2026 15:29:05 +0000 Subject: [PATCH 1/7] compiler: Substitute the object of a method Call in Uxreplace A Call through an object, e.g. `q.memcpy(...)`, keeps its callee only as a string, so Uxreplace could not rename `q`. reuse_efuncs then saw two Callables differing only in that object as distinct, and kept both. Keep the callee object on the Call and let Uxreplace substitute into it. --- devito/ir/iet/nodes.py | 3 +++ devito/ir/iet/visitors.py | 13 +++++++++---- tests/test_iet.py | 38 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/devito/ir/iet/nodes.py b/devito/ir/iet/nodes.py index 7c12b39647..14afed7be6 100644 --- a/devito/ir/iet/nodes.py +++ b/devito/ir/iet/nodes.py @@ -322,6 +322,9 @@ def __init__(self, name, arguments=None, retobj=None, is_indirect=False, self.base = name.base else: self.base = None + # The called object: a str or, for a call through an object such as + # `obj.method(...)`, a CallFromPointer + self.callee = name self.name = str(name) self.arguments = as_tuple(arguments) self.retobj = retobj diff --git a/devito/ir/iet/visitors.py b/devito/ir/iet/visitors.py index 6186ff4e76..a76847ab95 100644 --- a/devito/ir/iet/visitors.py +++ b/devito/ir/iet/visitors.py @@ -1474,11 +1474,16 @@ def visit_Call(self, o): arguments.append(self._visit(i)) else: arguments.append(uxreplace(i, self.mapper)) + kwargs = {'arguments': arguments} + if o.base is not None: + # A call through an object, e.g. `obj.method(...)`, where `obj` + # may be in the mapper too + callee = uxreplace(o.callee, self.mapper) + if callee is not o.callee: + kwargs['name'] = callee if o.retobj is not None: - retobj = uxreplace(o.retobj, self.mapper) - return reuse_if_unchanged(o, arguments=arguments, retobj=retobj) - else: - return reuse_if_unchanged(o, arguments=arguments) + kwargs['retobj'] = uxreplace(o.retobj, self.mapper) + return reuse_if_unchanged(o, **kwargs) def visit_Lambda(self, o): body = self._visit(o.body) diff --git a/tests/test_iet.py b/tests/test_iet.py index 99a94ed24c..a5641946b4 100644 --- a/tests/test_iet.py +++ b/tests/test_iet.py @@ -19,8 +19,8 @@ from devito.passes.iet.engine import Graph, reuse_efuncs from devito.passes.iet.languages.C import CDataManager from devito.symbolics import ( - FLOAT, Byref, Class, FieldFromComposite, InlineIf, ListInitializer, Macro, SizeOf, - String + FLOAT, Byref, Class, FieldFromComposite, FieldFromPointer, InlineIf, ListInitializer, + Macro, SizeOf, String ) from devito.symbolics.extended_dtypes import c_complex from devito.tools import CustomDtype, as_tuple, dtype_to_ctype @@ -564,6 +564,40 @@ def make_foo(name, start): assert [i.name for i in calls] == ['foo0', 'foo0'] +def test_reuse_efuncs_call_through_object(): + """ + Callables that differ only in the object a method is called through, + as in `q0->wait()` vs `q1->wait()`, must map to the same efunc. + + No OSS backend emits such Calls, hence an IET-level test. Downstream, the + SYCL backend emits `queue.memcpy(...)` in its streaming threads, and two + streamed Functions used to get two otherwise identical thread functions. + """ + + def make_foo(name, q): + p = Pointer(name=f'p{q.name}') + return make_callable(name, List(body=[ + Call(FieldFromPointer('copy', q), [p]), + Call(FieldFromPointer('wait', q)) + ])) + + foo0 = make_foo('foo0', Pointer(name='q0')) + foo1 = make_foo('foo1', Pointer(name='q1')) + + root = make_callable('root', List(body=[ + Call(foo0.name, foo0.parameters), + Call(foo1.name, foo1.parameters) + ])) + efuncs = {i.name: i for i in (root, foo0, foo1)} + + efuncs = reuse_efuncs(root, efuncs) + + assert set(efuncs) == {'root', 'foo0'} + calls = FindNodes(Call).visit(efuncs[root.name]) + assert [i.name for i in calls] == ['foo0', 'foo0'] + assert [str(i.arguments[1]) for i in calls] == ['q0', 'q1'] + + def test_sorted_efuncs_natural_numbering(): names = ['kernel3', 'foo0', 'kernel11', 'kernel1', 'kernel2'] efuncs = [make_callable(name, List()) for name in names] From 1b23feffaba463044a159a629a01ff759ce741c1 Mon Sep 17 00:00:00 2001 From: Gerard Gorman Date: Mon, 26 Oct 2026 16:34:16 +0000 Subject: [PATCH 2/7] arch: Query the memory of Intel GPUs through xpu-smi The Intel GPU branch of get_gpu_info installed memory callbacks that always returned None, so Device.memavail() was None on Intel GPUs and anything budgeting device memory (e.g. DevitoPRO's layered snapshotting) could not work. Read `memory_physical_size_byte` and `memory_free_size_byte` from `xpu-smi discovery -d -j`, and keep returning None if xpu-smi is unavailable or fails. The callbacks take a physical device ID, as with nvidia-smi, so treat ZE_AFFINITY_MASK as a visible-devices variable when mapping an MPI rank to its device, and run xpu-smi without it, since xpu-smi otherwise numbers only the devices the mask leaves visible. --- devito/arch/archinfo.py | 28 +++++++++++++++++++++++----- tests/test_gpu_common.py | 18 ++++++++++++++++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/devito/arch/archinfo.py b/devito/arch/archinfo.py index c294f5def4..80eb9b188b 100644 --- a/devito/arch/archinfo.py +++ b/devito/arch/archinfo.py @@ -384,13 +384,30 @@ def cbk(deviceid=0): gpu_info = homogenise_gpus(list(gpu_infos.values())) - # Also attach callbacks to retrieve instantaneous memory info - # Now this should be done using xpu-smi but for some reason - # it throws a lot of weird errors in docker so skipping for now + # Also attach callbacks to retrieve instantaneous memory info. + # `deviceid` is a physical device ID, like `nvidia-smi`'s, while + # `xpu-smi` only numbers the devices left visible by `ZE_AFFINITY_MASK` + def xpusmi_mem_info(deviceid): + cmd = ['xpu-smi', 'discovery', '-d', str(deviceid), '-j'] + env = {k: v for k, v in os.environ.items() if k != 'ZE_AFFINITY_MASK'} + try: + proc = run(cmd, stdout=PIPE, stderr=DEVNULL, text=True, check=True, + env=env) + info = json.loads(proc.stdout) + total = int(info['memory_physical_size_byte']) + free = int(info['memory_free_size_byte']) + except (OSError, CalledProcessError, json.JSONDecodeError, + KeyError, ValueError): + return None + return {'total': total, 'free': free, 'used': total - free} + for i in ['total', 'free', 'used']: def make_cbk(i): def cbk(deviceid=0): - return None + try: + return xpusmi_mem_info(deviceid)[i] + except TypeError: + return None return cbk gpu_info[f'mem.{i}'] = make_cbk(i) @@ -499,7 +516,8 @@ def parse_product_arch(): 'CUDA_VISIBLE_DEVICES', 'NVIDIA_VISIBLE_DEVICES', 'ROCR_VISIBLE_DEVICES', - 'HIP_VISIBLE_DEVICES' + 'HIP_VISIBLE_DEVICES', + 'ZE_AFFINITY_MASK' ) diff --git a/tests/test_gpu_common.py b/tests/test_gpu_common.py index c28cc3e61e..b97200bc2d 100644 --- a/tests/test_gpu_common.py +++ b/tests/test_gpu_common.py @@ -35,6 +35,18 @@ def test_get_gpu_info(self): # as we're not really exploiting this info yet... pytest.xfail("Unsupported platform for get_gpu_info") + def test_memavail(self): + """ + The device memory budget of an Operator (e.g., for snapshotting) + needs the available memory of the target device. + """ + plat = configuration['platform'] + + memavail = plat.memavail() + + assert isinstance(memavail, int) + assert 0 < memavail <= plat.memtotal + def custom_compiler(self): grid = Grid(shape=(4, 4)) @@ -84,7 +96,8 @@ class TestDeviceID: 'CUDA_VISIBLE_DEVICES', 'NVIDIA_VISIBLE_DEVICES', 'ROCR_VISIBLE_DEVICES', - 'HIP_VISIBLE_DEVICES' + 'HIP_VISIBLE_DEVICES', + 'ZE_AFFINITY_MASK' ) @pytest.mark.parametrize('env_variables', [{"CUDA_VISIBLE_DEVICES": "1"}, @@ -93,7 +106,8 @@ class TestDeviceID: {"NVIDIA_VISIBLE_DEVICES": "1"}, {"NVIDIA_VISIBLE_DEVICES": "1,2"}, {"ROCR_VISIBLE_DEVICES": "1"}, - {"HIP_VISIBLE_DEVICES": " 1"}]) + {"HIP_VISIBLE_DEVICES": " 1"}, + {"ZE_AFFINITY_MASK": "1"}]) def test_visible_devices(self, env_variables): """ Test that physical device IDs used for querying memory on a device via From 8affde8df279f9ae595394f37b696474a03db8ea Mon Sep 17 00:00:00 2001 From: Gerard Gorman Date: Mon, 26 Oct 2026 15:48:08 +0000 Subject: [PATCH 3/7] docker: Take the Intel GPU runtime from the LTS 2523 channel The `graphics/ubuntu jammy unified` channel stops at compute-runtime 23.17. With it, the oneAPI 2026 compilers find no Level Zero device on a Data Center GPU Max 1100 and OpenMP offload images fail to load. Install the Max-series user-space stack (compute-runtime 25.18, IGC 2.11, Level Zero loader 1.21+) from `gpu/ubuntu jammy/lts/2523`, under its current package names, and add a `gpu-icx` target for OpenMP offloading to Intel GPUs. --- docker/Dockerfile.intel | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.intel b/docker/Dockerfile.intel index c7a7a0f071..ba09678c10 100644 --- a/docker/Dockerfile.intel +++ b/docker/Dockerfile.intel @@ -37,13 +37,17 @@ RUN apt-get update -y && \ # Drivers mandatory for intel gpu # https://dgpu-docs.intel.com/driver/installation.html#ubuntu-install-steps +# NOTE: The Data Center GPU Max (PVC) user-space stack comes from the LTS channels. +# The legacy `graphics/ubuntu jammy unified` channel stops at compute-runtime 23.17, +# with which the oneAPI 2026 compilers cannot drive a Max 1100 (SYCL finds no Level +# Zero device and OpenMP offload images fail to load) SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN wget -qO - https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor > /usr/share/keyrings/intel-graphics.gpg -RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy unified" > /etc/apt/sources.list.d/intel-gpu-jammy.list +RUN wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --dearmor > /usr/share/keyrings/intel-graphics.gpg +RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy/lts/2523 unified" > /etc/apt/sources.list.d/intel-gpu-jammy.list RUN apt-get update -y && apt-get dist-upgrade -y && \ # Compute and media runtime - apt-get install -y intel-opencl-icd intel-level-zero-gpu level-zero \ + apt-get install -y intel-opencl-icd libze-intel-gpu1 libze1 \ intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \ libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \ libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \ @@ -51,7 +55,7 @@ RUN apt-get update -y && apt-get dist-upgrade -y && \ # XPU-SMI pciutils libkmod-dev intel-fw-gpu xpu-smi \ # Development packages - libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev level-zero-dev + libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev libze-dev ENV MPI4PY_FLAGS='. /opt/intel/oneapi/setvars.sh intel64 && ' ENV MPI4PY_RC_RECV_MPROBE=0 @@ -104,6 +108,14 @@ ENV DEVITO_ARCH="sycl" ENV DEVITO_LANGUAGE="sycl" ENV DEVITO_PLATFORM="intel64" +############################################################## +# ICX OpenMP GPU image +############################################################## +FROM icx AS gpu-icx + +# Devito config +ENV DEVITO_PLATFORM="intelgpuX" + ############################################################## # ICX SYCL GPU image ############################################################## From 2db0029e359a531e1395674e702da0a85f53284c Mon Sep 17 00:00:00 2001 From: Gerard Gorman Date: Mon, 26 Oct 2026 15:48:08 +0000 Subject: [PATCH 4/7] compiler: Stop passing -fopenmp-target-simd for Intel GPUs With oneAPI 2026.0 on a Data Center GPU Max 1100 the explicit-SIMD device compilation fails at load time for every Devito Operator: IGC 1.0.15468 (LTS 2350) rejects the image ("SLM variable usage in indirectly called functions is not supported") and IGC 2.11 (LTS 2523, the newest for Ubuntu 22.04) crashes. Without the flag, the GPU test files pass. --- devito/arch/compiler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/devito/arch/compiler.py b/devito/arch/compiler.py index d356765636..1b8308763f 100644 --- a/devito/arch/compiler.py +++ b/devito/arch/compiler.py @@ -963,7 +963,6 @@ def __init_finalize__(self, **kwargs): elif isinstance(platform, IntelDevice): self.cflags.append('-fiopenmp') self.cflags.append('-fopenmp-targets=spir64') - self.cflags.append('-fopenmp-target-simd') self.cflags.remove('-g') # -g disables some optimizations in IGC self.cflags.append('-gline-tables-only') From 19f150e162a771fad358f457b88df3bd91aee375 Mon Sep 17 00:00:00 2001 From: Gerard Gorman Date: Mon, 26 Oct 2026 15:48:08 +0000 Subject: [PATCH 5/7] ci: Run the GPU tests on Intel GPUs with OpenMP offloading Add a pytest-gpu-omp-intel job, on self-hosted runners labelled INTELGPU, running the same test files and examples as the AMD OpenMP job. The runner selects its device through ZE_AFFINITY_MASK. --- .github/workflows/pytest-gpu.yaml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index 9bf17dab28..85215493f1 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -1,6 +1,7 @@ # Workflow information: # - OpenACC/OpenMP on NVIDIA runs on runners labeled `nvidiagpu` # - OpenMP on AMD runs on runners labeled `amdgpu` +# - OpenMP on Intel runs on runners labeled `INTELGPU`; ZE_AFFINITY_MASK selects the device # - Respect CUDA_VISIBLE_DEVICES for NVIDIA jobs by passing it AND restricting Docker with --gpus "device=…" # - Tag images and container names to avoid cross-runner races and maximize cache reuse @@ -36,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - name: [pytest-gpu-acc-nvidia, pytest-gpu-omp-amd] + name: [pytest-gpu-acc-nvidia, pytest-gpu-omp-amd, pytest-gpu-omp-intel] test_examples: ["examples/seismic/tti/tti_example.py examples/seismic/acoustic/acoustic_example.py examples/seismic/viscoacoustic/viscoacoustic_example.py examples/seismic/viscoelastic/viscoelastic_example.py examples/seismic/elastic/elastic_example.py"] include: @@ -69,6 +70,22 @@ jobs: --group-add video --group-add "$(getent group render | cut -d: -f3)" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined + # -------------------- Intel job --------------------- + - name: pytest-gpu-omp-intel + test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py tests/test_operator.py::TestEstimateMemory" + runner_label: INTELGPU + base: "devitocodes/bases:gpu-icx" + test_drive_cmd: "sycl-ls" + # Respect ZE_AFFINITY_MASK, set by the runner like CUDA_VISIBLE_DEVICES + dockerflags: >- + --cpuset-cpus="${AVAILABLE_CPUSET}" + --memory="${AVAILABLE_MEM}" + --memory-swap="${AVAILABLE_MEM}" + -v /dev/dri/by-path:/dev/dri/by-path + --device=/dev/dri:/dev/dri + --group-add "$(getent group render | cut -d: -f3)" + --env ZE_AFFINITY_MASK="${ZE_AFFINITY_MASK}" + steps: - name: Checkout devito uses: actions/checkout@v7 From 8af16d5e3556cce12e068b2ed7a3b269b885f0cf Mon Sep 17 00:00:00 2001 From: Gerard Gorman Date: Mon, 26 Oct 2026 15:48:08 +0000 Subject: [PATCH 6/7] ci: Build the Intel base images again, including gpu-icx Undo the switch-off of c57bf01f7 so that the LTS 2523 runtime reaches devitocodes/bases:{cpu-icx,cpu-sycl,gpu-sycl}, and build the new gpu-icx image the Intel GPU job uses. --- .github/workflows/docker-bases.yaml | 118 +++++++++++++++------------- 1 file changed, 64 insertions(+), 54 deletions(-) diff --git a/.github/workflows/docker-bases.yaml b/.github/workflows/docker-bases.yaml index bc5fa51b71..1fa76aa9ac 100644 --- a/.github/workflows/docker-bases.yaml +++ b/.github/workflows/docker-bases.yaml @@ -122,60 +122,70 @@ jobs: # ####################################################### # ############## Intel OneApi CPU ####################### # ####################################################### - # deploy-oneapi-bases: - # if: ${{ github.event_name != 'workflow_dispatch' || inputs.intel }} - # name: "oneapi-base" - # runs-on: ubuntu-latest - # env: - # DOCKER_BUILDKIT: "1" - - # steps: - # - name: Checkout devito - # uses: actions/checkout@v7 - - # - name: Check event name - # run: echo ${{ github.event_name }} - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v4 - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v4 - - # - name: Login to DockerHub - # uses: docker/login-action@v4 - # with: - # username: ${{ secrets.DOCKER_USERNAME }} - # password: ${{ secrets.DOCKER_PASSWORD }} - # - name: ICX image - # uses: docker/build-push-action@v7 - # with: - # context: . - # file: "./docker/Dockerfile.intel" - # push: true - # target: "icx" - # build-args: "arch=icx" - # tags: "devitocodes/bases:cpu-icx" - - # - name: SYCL CPU image - # uses: docker/build-push-action@v7 - # with: - # context: . - # file: "./docker/Dockerfile.intel" - # push: true - # target: "cpu-sycl" - # build-args: "arch=cpu-sycl" - # tags: "devitocodes/bases:cpu-sycl" - - # - name: SYCL GPU image - # uses: docker/build-push-action@v7 - # with: - # context: . - # file: "./docker/Dockerfile.intel" - # push: true - # target: "gpu-sycl" - # build-args: "arch=gpu-sycl" - # tags: "devitocodes/bases:gpu-sycl" + deploy-oneapi-bases: + if: ${{ github.event_name != 'workflow_dispatch' || inputs.intel }} + name: "oneapi-base" + runs-on: ubuntu-latest + env: + DOCKER_BUILDKIT: "1" + + steps: + - name: Checkout devito + uses: actions/checkout@v7 + + - name: Check event name + run: echo ${{ github.event_name }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: ICX image + uses: docker/build-push-action@v7 + with: + context: . + file: "./docker/Dockerfile.intel" + push: true + target: "icx" + build-args: "arch=icx" + tags: "devitocodes/bases:cpu-icx" + + - name: SYCL CPU image + uses: docker/build-push-action@v7 + with: + context: . + file: "./docker/Dockerfile.intel" + push: true + target: "cpu-sycl" + build-args: "arch=cpu-sycl" + tags: "devitocodes/bases:cpu-sycl" + + - name: OpenMP GPU image + uses: docker/build-push-action@v7 + with: + context: . + file: "./docker/Dockerfile.intel" + push: true + target: "gpu-icx" + build-args: "arch=gpu-icx" + tags: "devitocodes/bases:gpu-icx" + + - name: SYCL GPU image + uses: docker/build-push-action@v7 + with: + context: . + file: "./docker/Dockerfile.intel" + push: true + target: "gpu-sycl" + build-args: "arch=gpu-sycl" + tags: "devitocodes/bases:gpu-sycl" ####################################################### ################### Nvidia nvhpc ###################### From 141bd8d7aebddfd2dff60c2e3b0d7e42eff65dfe Mon Sep 17 00:00:00 2001 From: Gerard Gorman Date: Mon, 26 Oct 2026 21:01:37 +0000 Subject: [PATCH 7/7] docker: Let Intel MPI spawn processes in the Intel images Intel MPI 2021.19 picks the shm fabric alone on a single node, and shm refuses MPI_Comm_spawn, so the DevitoPRO decoupler tests hung in the rebuilt images. Use shm:ofi. --- docker/Dockerfile.intel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile.intel b/docker/Dockerfile.intel index ba09678c10..7d1eca8a01 100644 --- a/docker/Dockerfile.intel +++ b/docker/Dockerfile.intel @@ -59,6 +59,9 @@ RUN apt-get update -y && apt-get dist-upgrade -y && \ ENV MPI4PY_FLAGS='. /opt/intel/oneapi/setvars.sh intel64 && ' ENV MPI4PY_RC_RECV_MPROBE=0 +# Intel MPI 2021.19 picks the `shm` fabric alone on a single node, which +# refuses `MPI_Comm_spawn` (as used by the DevitoPRO decoupler) +ENV I_MPI_FABRICS="shm:ofi" RUN apt-get clean && apt-get autoclean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*