ci: Test OpenMP offloading on Intel GPUs, with the Devito fixes it needs

.github/workflows/docker-bases.yaml

@@ -122,60 +122,70 @@ jobs:
   # #######################################################
   # ############## Intel OneApi CPU #######################
   # #######################################################
-  # deploy-oneapi-bases:
-  #   if: ${{ github.event_name != 'workflow_dispatch' || inputs.intel }}
-  #   name: "oneapi-base"
-  #   runs-on: ubuntu-latest
-  #   env:
-  #     DOCKER_BUILDKIT: "1"
-
-  #   steps:
-  #     - name: Checkout devito
-  #       uses: actions/checkout@v7
-
-  #     - name: Check event name
-  #       run: echo ${{ github.event_name }}
-
-  #     - name: Set up QEMU
-  #       uses: docker/setup-qemu-action@v4
-
-  #     - name: Set up Docker Buildx
-  #       uses: docker/setup-buildx-action@v4
-
-  #     - name: Login to DockerHub
-  #       uses: docker/login-action@v4
-  #       with:
-  #         username: ${{ secrets.DOCKER_USERNAME }}
-  #         password: ${{ secrets.DOCKER_PASSWORD }}
-  #     - name: ICX image
-  #       uses: docker/build-push-action@v7
-  #       with:
-  #         context: .
-  #         file: "./docker/Dockerfile.intel"
-  #         push: true
-  #         target: "icx"
-  #         build-args: "arch=icx"
-  #         tags: "devitocodes/bases:cpu-icx"
-
-  #     - name: SYCL CPU image
-  #       uses: docker/build-push-action@v7
-  #       with:
-  #         context: .
-  #         file: "./docker/Dockerfile.intel"
-  #         push: true
-  #         target: "cpu-sycl"
-  #         build-args: "arch=cpu-sycl"
-  #         tags: "devitocodes/bases:cpu-sycl"
-
-  #     - name: SYCL GPU image
-  #       uses: docker/build-push-action@v7
-  #       with:
-  #         context: .
-  #         file: "./docker/Dockerfile.intel"
-  #         push: true
-  #         target: "gpu-sycl"
-  #         build-args: "arch=gpu-sycl"
-  #         tags: "devitocodes/bases:gpu-sycl"
+  deploy-oneapi-bases:
+    if: ${{ github.event_name != 'workflow_dispatch' || inputs.intel }}
+    name: "oneapi-base"
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: "1"
+
+    steps:
+      - name: Checkout devito
+        uses: actions/checkout@v7
+
+      - name: Check event name
+        run: echo ${{ github.event_name }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: ICX image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: "./docker/Dockerfile.intel"
+          push: true
+          target: "icx"
+          build-args: "arch=icx"
+          tags: "devitocodes/bases:cpu-icx"
+
+      - name: SYCL CPU image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: "./docker/Dockerfile.intel"
+          push: true
+          target: "cpu-sycl"
+          build-args: "arch=cpu-sycl"
+          tags: "devitocodes/bases:cpu-sycl"
+
+      - name: OpenMP GPU image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: "./docker/Dockerfile.intel"
+          push: true
+          target: "gpu-icx"
+          build-args: "arch=gpu-icx"
+          tags: "devitocodes/bases:gpu-icx"
+
+      - name: SYCL GPU image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: "./docker/Dockerfile.intel"
+          push: true
+          target: "gpu-sycl"
+          build-args: "arch=gpu-sycl"
+          tags: "devitocodes/bases:gpu-sycl"
 
   #######################################################
   ################### Nvidia nvhpc ######################

.github/workflows/pytest-gpu.yaml

@@ -1,6 +1,7 @@
 # Workflow information:
 #  - OpenACC/OpenMP on NVIDIA runs on runners labeled `nvidiagpu`
 #  - OpenMP on AMD runs on runners labeled `amdgpu`
+#  - OpenMP on Intel runs on runners labeled `INTELGPU`; ZE_AFFINITY_MASK selects the device
 #  - Respect CUDA_VISIBLE_DEVICES for NVIDIA jobs by passing it AND restricting Docker with --gpus "device=…"
 #  - Tag images and container names to avoid cross-runner races and maximize cache reuse
 
@@ -36,7 +37,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        name: [pytest-gpu-acc-nvidia, pytest-gpu-omp-amd]
+        name: [pytest-gpu-acc-nvidia, pytest-gpu-omp-amd, pytest-gpu-omp-intel]
         test_examples: ["examples/seismic/tti/tti_example.py examples/seismic/acoustic/acoustic_example.py examples/seismic/viscoacoustic/viscoacoustic_example.py examples/seismic/viscoelastic/viscoelastic_example.py examples/seismic/elastic/elastic_example.py"]
 
         include:
@@ -69,6 +70,22 @@ jobs:
             --group-add video --group-add "$(getent group render | cut -d: -f3)"
             --cap-add=SYS_PTRACE --security-opt seccomp=unconfined
 
+        # -------------------- Intel job ---------------------
+        - name: pytest-gpu-omp-intel
+          test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py tests/test_operator.py::TestEstimateMemory"
+          runner_label: INTELGPU
+          base: "devitocodes/bases:gpu-icx"
+          test_drive_cmd: "sycl-ls"
+          # Respect ZE_AFFINITY_MASK, set by the runner like CUDA_VISIBLE_DEVICES
+          dockerflags: >-
+            --cpuset-cpus="${AVAILABLE_CPUSET}"
+            --memory="${AVAILABLE_MEM}"
+            --memory-swap="${AVAILABLE_MEM}"
+            -v /dev/dri/by-path:/dev/dri/by-path
+            --device=/dev/dri:/dev/dri
+            --group-add "$(getent group render | cut -d: -f3)"
+            --env ZE_AFFINITY_MASK="${ZE_AFFINITY_MASK}"
+
     steps:
     - name: Checkout devito
       uses: actions/checkout@v7

devito/arch/archinfo.py

@@ -384,13 +384,30 @@ def cbk(deviceid=0):
 
         gpu_info = homogenise_gpus(list(gpu_infos.values()))
 
-        # Also attach callbacks to retrieve instantaneous memory info
-        # Now this should be done using xpu-smi but for some reason
-        # it throws a lot of weird errors in docker so skipping for now
+        # Also attach callbacks to retrieve instantaneous memory info.
+        # `deviceid` is a physical device ID, like `nvidia-smi`'s, while
+        # `xpu-smi` only numbers the devices left visible by `ZE_AFFINITY_MASK`
+        def xpusmi_mem_info(deviceid):
+            cmd = ['xpu-smi', 'discovery', '-d', str(deviceid), '-j']
+            env = {k: v for k, v in os.environ.items() if k != 'ZE_AFFINITY_MASK'}
+            try:
+                proc = run(cmd, stdout=PIPE, stderr=DEVNULL, text=True, check=True,
+                           env=env)
+                info = json.loads(proc.stdout)
+                total = int(info['memory_physical_size_byte'])
+                free = int(info['memory_free_size_byte'])
+            except (OSError, CalledProcessError, json.JSONDecodeError,
+                    KeyError, ValueError):
+                return None
+            return {'total': total, 'free': free, 'used': total - free}
+
         for i in ['total', 'free', 'used']:
             def make_cbk(i):
                 def cbk(deviceid=0):
-                    return None
+                    try:
+                        return xpusmi_mem_info(deviceid)[i]
+                    except TypeError:
+                        return None
                 return cbk
 
             gpu_info[f'mem.{i}'] = make_cbk(i)
@@ -499,7 +516,8 @@ def parse_product_arch():
     'CUDA_VISIBLE_DEVICES',
     'NVIDIA_VISIBLE_DEVICES',
     'ROCR_VISIBLE_DEVICES',
-    'HIP_VISIBLE_DEVICES'
+    'HIP_VISIBLE_DEVICES',
+    'ZE_AFFINITY_MASK'
 )
 
 

devito/arch/compiler.py

@@ -963,7 +963,6 @@ def __init_finalize__(self, **kwargs):
             elif isinstance(platform, IntelDevice):
                 self.cflags.append('-fiopenmp')
                 self.cflags.append('-fopenmp-targets=spir64')
-                self.cflags.append('-fopenmp-target-simd')
 
                 self.cflags.remove('-g')  # -g disables some optimizations in IGC
                 self.cflags.append('-gline-tables-only')

devito/ir/iet/nodes.py

@@ -322,6 +322,9 @@ def __init__(self, name, arguments=None, retobj=None, is_indirect=False,
             self.base = name.base
         else:
             self.base = None
+        # The called object: a str or, for a call through an object such as
+        # `obj.method(...)`, a CallFromPointer
+        self.callee = name
         self.name = str(name)
         self.arguments = as_tuple(arguments)
         self.retobj = retobj

devito/ir/iet/visitors.py

@@ -1474,11 +1474,16 @@ def visit_Call(self, o):
                 arguments.append(self._visit(i))
             else:
                 arguments.append(uxreplace(i, self.mapper))
+        kwargs = {'arguments': arguments}
+        if o.base is not None:
+            # A call through an object, e.g. `obj.method(...)`, where `obj`
+            # may be in the mapper too
+            callee = uxreplace(o.callee, self.mapper)
+            if callee is not o.callee:
+                kwargs['name'] = callee
         if o.retobj is not None:
-            retobj = uxreplace(o.retobj, self.mapper)
-            return reuse_if_unchanged(o, arguments=arguments, retobj=retobj)
-        else:
-            return reuse_if_unchanged(o, arguments=arguments)
+            kwargs['retobj'] = uxreplace(o.retobj, self.mapper)
+        return reuse_if_unchanged(o, **kwargs)
 
     def visit_Lambda(self, o):
         body = self._visit(o.body)

docker/Dockerfile.intel

@@ -37,24 +37,31 @@ RUN apt-get update -y && \
 
 # Drivers mandatory for intel gpu
 # https://dgpu-docs.intel.com/driver/installation.html#ubuntu-install-steps
+# NOTE: The Data Center GPU Max (PVC) user-space stack comes from the LTS channels.
+# The legacy `graphics/ubuntu jammy unified` channel stops at compute-runtime 23.17,
+# with which the oneAPI 2026 compilers cannot drive a Max 1100 (SYCL finds no Level
+# Zero device and OpenMP offload images fail to load)
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN wget -qO - https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor > /usr/share/keyrings/intel-graphics.gpg
-RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy unified" >  /etc/apt/sources.list.d/intel-gpu-jammy.list
+RUN wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --dearmor > /usr/share/keyrings/intel-graphics.gpg
+RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy/lts/2523 unified" >  /etc/apt/sources.list.d/intel-gpu-jammy.list
 
 RUN apt-get update -y && apt-get dist-upgrade -y && \
     # Compute and media runtime
-    apt-get install -y intel-opencl-icd intel-level-zero-gpu level-zero \
+    apt-get install -y intel-opencl-icd libze-intel-gpu1 libze1 \
             intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \
             libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \
             libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \
             mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo \
             # XPU-SMI
             pciutils libkmod-dev intel-fw-gpu xpu-smi \
             # Development packages
-            libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev level-zero-dev
+            libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev libze-dev
 
 ENV MPI4PY_FLAGS='. /opt/intel/oneapi/setvars.sh intel64 && '
 ENV MPI4PY_RC_RECV_MPROBE=0
+# Intel MPI 2021.19 picks the `shm` fabric alone on a single node, which
+# refuses `MPI_Comm_spawn` (as used by the DevitoPRO decoupler)
+ENV I_MPI_FABRICS="shm:ofi"
 
 RUN apt-get clean && apt-get autoclean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
 
@@ -104,6 +111,14 @@ ENV DEVITO_ARCH="sycl"
 ENV DEVITO_LANGUAGE="sycl"
 ENV DEVITO_PLATFORM="intel64"
 
+##############################################################
+# ICX OpenMP GPU image
+##############################################################
+FROM icx AS gpu-icx
+
+# Devito config
+ENV DEVITO_PLATFORM="intelgpuX"
+
 ##############################################################
 # ICX SYCL GPU image
 ##############################################################

tests/test_gpu_common.py

@@ -35,6 +35,18 @@ def test_get_gpu_info(self):
             # as we're not really exploiting this info yet...
             pytest.xfail("Unsupported platform for get_gpu_info")
 
+    def test_memavail(self):
+        """
+        The device memory budget of an Operator (e.g., for snapshotting)
+        needs the available memory of the target device.
+        """
+        plat = configuration['platform']
+
+        memavail = plat.memavail()
+
+        assert isinstance(memavail, int)
+        assert 0 < memavail <= plat.memtotal
+
     def custom_compiler(self):
         grid = Grid(shape=(4, 4))
 
@@ -84,7 +96,8 @@ class TestDeviceID:
         'CUDA_VISIBLE_DEVICES',
         'NVIDIA_VISIBLE_DEVICES',
         'ROCR_VISIBLE_DEVICES',
-        'HIP_VISIBLE_DEVICES'
+        'HIP_VISIBLE_DEVICES',
+        'ZE_AFFINITY_MASK'
     )
 
     @pytest.mark.parametrize('env_variables', [{"CUDA_VISIBLE_DEVICES": "1"},
@@ -93,7 +106,8 @@ class TestDeviceID:
                                                {"NVIDIA_VISIBLE_DEVICES": "1"},
                                                {"NVIDIA_VISIBLE_DEVICES": "1,2"},
                                                {"ROCR_VISIBLE_DEVICES": "1"},
-                                               {"HIP_VISIBLE_DEVICES": " 1"}])
+                                               {"HIP_VISIBLE_DEVICES": " 1"},
+                                               {"ZE_AFFINITY_MASK": "1"}])
     def test_visible_devices(self, env_variables):
         """
         Test that physical device IDs used for querying memory on a device via

tests/test_iet.py

@@ -19,8 +19,8 @@
 from devito.passes.iet.engine import Graph, reuse_efuncs
 from devito.passes.iet.languages.C import CDataManager
 from devito.symbolics import (
-    FLOAT, Byref, Class, FieldFromComposite, InlineIf, ListInitializer, Macro, SizeOf,
-    String
+    FLOAT, Byref, Class, FieldFromComposite, FieldFromPointer, InlineIf, ListInitializer,
+    Macro, SizeOf, String
 )
 from devito.symbolics.extended_dtypes import c_complex
 from devito.tools import CustomDtype, as_tuple, dtype_to_ctype
@@ -564,6 +564,40 @@ def make_foo(name, start):
     assert [i.name for i in calls] == ['foo0', 'foo0']
 
 
+def test_reuse_efuncs_call_through_object():
+    """
+    Callables that differ only in the object a method is called through,
+    as in `q0->wait()` vs `q1->wait()`, must map to the same efunc.
+
+    No OSS backend emits such Calls, hence an IET-level test. Downstream, the
+    SYCL backend emits `queue.memcpy(...)` in its streaming threads, and two
+    streamed Functions used to get two otherwise identical thread functions.
+    """
+
+    def make_foo(name, q):
+        p = Pointer(name=f'p{q.name}')
+        return make_callable(name, List(body=[
+            Call(FieldFromPointer('copy', q), [p]),
+            Call(FieldFromPointer('wait', q))
+        ]))
+
+    foo0 = make_foo('foo0', Pointer(name='q0'))
+    foo1 = make_foo('foo1', Pointer(name='q1'))
+
+    root = make_callable('root', List(body=[
+        Call(foo0.name, foo0.parameters),
+        Call(foo1.name, foo1.parameters)
+    ]))
+    efuncs = {i.name: i for i in (root, foo0, foo1)}
+
+    efuncs = reuse_efuncs(root, efuncs)
+
+    assert set(efuncs) == {'root', 'foo0'}
+    calls = FindNodes(Call).visit(efuncs[root.name])
+    assert [i.name for i in calls] == ['foo0', 'foo0']
+    assert [str(i.arguments[1]) for i in calls] == ['q0', 'q1']
+
+
 def test_sorted_efuncs_natural_numbering():
     names = ['kernel3', 'foo0', 'kernel11', 'kernel1', 'kernel2']
     efuncs = [make_callable(name, List()) for name in names]