diff --git a/.github/workflows/docker-bases.yaml b/.github/workflows/docker-bases.yaml index bc5fa51b71..1fa76aa9ac 100644 --- a/.github/workflows/docker-bases.yaml +++ b/.github/workflows/docker-bases.yaml @@ -122,60 +122,70 @@ jobs: # ####################################################### # ############## Intel OneApi CPU ####################### # ####################################################### - # deploy-oneapi-bases: - # if: ${{ github.event_name != 'workflow_dispatch' || inputs.intel }} - # name: "oneapi-base" - # runs-on: ubuntu-latest - # env: - # DOCKER_BUILDKIT: "1" - - # steps: - # - name: Checkout devito - # uses: actions/checkout@v7 - - # - name: Check event name - # run: echo ${{ github.event_name }} - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v4 - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v4 - - # - name: Login to DockerHub - # uses: docker/login-action@v4 - # with: - # username: ${{ secrets.DOCKER_USERNAME }} - # password: ${{ secrets.DOCKER_PASSWORD }} - # - name: ICX image - # uses: docker/build-push-action@v7 - # with: - # context: . - # file: "./docker/Dockerfile.intel" - # push: true - # target: "icx" - # build-args: "arch=icx" - # tags: "devitocodes/bases:cpu-icx" - - # - name: SYCL CPU image - # uses: docker/build-push-action@v7 - # with: - # context: . - # file: "./docker/Dockerfile.intel" - # push: true - # target: "cpu-sycl" - # build-args: "arch=cpu-sycl" - # tags: "devitocodes/bases:cpu-sycl" - - # - name: SYCL GPU image - # uses: docker/build-push-action@v7 - # with: - # context: . - # file: "./docker/Dockerfile.intel" - # push: true - # target: "gpu-sycl" - # build-args: "arch=gpu-sycl" - # tags: "devitocodes/bases:gpu-sycl" + deploy-oneapi-bases: + if: ${{ github.event_name != 'workflow_dispatch' || inputs.intel }} + name: "oneapi-base" + runs-on: ubuntu-latest + env: + DOCKER_BUILDKIT: "1" + + steps: + - name: Checkout devito + uses: actions/checkout@v7 + + - name: Check event name + run: echo ${{ github.event_name }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: ICX image + uses: docker/build-push-action@v7 + with: + context: . + file: "./docker/Dockerfile.intel" + push: true + target: "icx" + build-args: "arch=icx" + tags: "devitocodes/bases:cpu-icx" + + - name: SYCL CPU image + uses: docker/build-push-action@v7 + with: + context: . + file: "./docker/Dockerfile.intel" + push: true + target: "cpu-sycl" + build-args: "arch=cpu-sycl" + tags: "devitocodes/bases:cpu-sycl" + + - name: OpenMP GPU image + uses: docker/build-push-action@v7 + with: + context: . + file: "./docker/Dockerfile.intel" + push: true + target: "gpu-icx" + build-args: "arch=gpu-icx" + tags: "devitocodes/bases:gpu-icx" + + - name: SYCL GPU image + uses: docker/build-push-action@v7 + with: + context: . + file: "./docker/Dockerfile.intel" + push: true + target: "gpu-sycl" + build-args: "arch=gpu-sycl" + tags: "devitocodes/bases:gpu-sycl" ####################################################### ################### Nvidia nvhpc ###################### diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index 9bf17dab28..85215493f1 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -1,6 +1,7 @@ # Workflow information: # - OpenACC/OpenMP on NVIDIA runs on runners labeled `nvidiagpu` # - OpenMP on AMD runs on runners labeled `amdgpu` +# - OpenMP on Intel runs on runners labeled `INTELGPU`; ZE_AFFINITY_MASK selects the device # - Respect CUDA_VISIBLE_DEVICES for NVIDIA jobs by passing it AND restricting Docker with --gpus "device=…" # - Tag images and container names to avoid cross-runner races and maximize cache reuse @@ -36,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - name: [pytest-gpu-acc-nvidia, pytest-gpu-omp-amd] + name: [pytest-gpu-acc-nvidia, pytest-gpu-omp-amd, pytest-gpu-omp-intel] test_examples: ["examples/seismic/tti/tti_example.py examples/seismic/acoustic/acoustic_example.py examples/seismic/viscoacoustic/viscoacoustic_example.py examples/seismic/viscoelastic/viscoelastic_example.py examples/seismic/elastic/elastic_example.py"] include: @@ -69,6 +70,22 @@ jobs: --group-add video --group-add "$(getent group render | cut -d: -f3)" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined + # -------------------- Intel job --------------------- + - name: pytest-gpu-omp-intel + test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py tests/test_operator.py::TestEstimateMemory" + runner_label: INTELGPU + base: "devitocodes/bases:gpu-icx" + test_drive_cmd: "sycl-ls" + # Respect ZE_AFFINITY_MASK, set by the runner like CUDA_VISIBLE_DEVICES + dockerflags: >- + --cpuset-cpus="${AVAILABLE_CPUSET}" + --memory="${AVAILABLE_MEM}" + --memory-swap="${AVAILABLE_MEM}" + -v /dev/dri/by-path:/dev/dri/by-path + --device=/dev/dri:/dev/dri + --group-add "$(getent group render | cut -d: -f3)" + --env ZE_AFFINITY_MASK="${ZE_AFFINITY_MASK}" + steps: - name: Checkout devito uses: actions/checkout@v7 diff --git a/devito/arch/archinfo.py b/devito/arch/archinfo.py index c294f5def4..80eb9b188b 100644 --- a/devito/arch/archinfo.py +++ b/devito/arch/archinfo.py @@ -384,13 +384,30 @@ def cbk(deviceid=0): gpu_info = homogenise_gpus(list(gpu_infos.values())) - # Also attach callbacks to retrieve instantaneous memory info - # Now this should be done using xpu-smi but for some reason - # it throws a lot of weird errors in docker so skipping for now + # Also attach callbacks to retrieve instantaneous memory info. + # `deviceid` is a physical device ID, like `nvidia-smi`'s, while + # `xpu-smi` only numbers the devices left visible by `ZE_AFFINITY_MASK` + def xpusmi_mem_info(deviceid): + cmd = ['xpu-smi', 'discovery', '-d', str(deviceid), '-j'] + env = {k: v for k, v in os.environ.items() if k != 'ZE_AFFINITY_MASK'} + try: + proc = run(cmd, stdout=PIPE, stderr=DEVNULL, text=True, check=True, + env=env) + info = json.loads(proc.stdout) + total = int(info['memory_physical_size_byte']) + free = int(info['memory_free_size_byte']) + except (OSError, CalledProcessError, json.JSONDecodeError, + KeyError, ValueError): + return None + return {'total': total, 'free': free, 'used': total - free} + for i in ['total', 'free', 'used']: def make_cbk(i): def cbk(deviceid=0): - return None + try: + return xpusmi_mem_info(deviceid)[i] + except TypeError: + return None return cbk gpu_info[f'mem.{i}'] = make_cbk(i) @@ -499,7 +516,8 @@ def parse_product_arch(): 'CUDA_VISIBLE_DEVICES', 'NVIDIA_VISIBLE_DEVICES', 'ROCR_VISIBLE_DEVICES', - 'HIP_VISIBLE_DEVICES' + 'HIP_VISIBLE_DEVICES', + 'ZE_AFFINITY_MASK' ) diff --git a/devito/arch/compiler.py b/devito/arch/compiler.py index d356765636..1b8308763f 100644 --- a/devito/arch/compiler.py +++ b/devito/arch/compiler.py @@ -963,7 +963,6 @@ def __init_finalize__(self, **kwargs): elif isinstance(platform, IntelDevice): self.cflags.append('-fiopenmp') self.cflags.append('-fopenmp-targets=spir64') - self.cflags.append('-fopenmp-target-simd') self.cflags.remove('-g') # -g disables some optimizations in IGC self.cflags.append('-gline-tables-only') diff --git a/devito/ir/iet/nodes.py b/devito/ir/iet/nodes.py index 7c12b39647..14afed7be6 100644 --- a/devito/ir/iet/nodes.py +++ b/devito/ir/iet/nodes.py @@ -322,6 +322,9 @@ def __init__(self, name, arguments=None, retobj=None, is_indirect=False, self.base = name.base else: self.base = None + # The called object: a str or, for a call through an object such as + # `obj.method(...)`, a CallFromPointer + self.callee = name self.name = str(name) self.arguments = as_tuple(arguments) self.retobj = retobj diff --git a/devito/ir/iet/visitors.py b/devito/ir/iet/visitors.py index 6186ff4e76..a76847ab95 100644 --- a/devito/ir/iet/visitors.py +++ b/devito/ir/iet/visitors.py @@ -1474,11 +1474,16 @@ def visit_Call(self, o): arguments.append(self._visit(i)) else: arguments.append(uxreplace(i, self.mapper)) + kwargs = {'arguments': arguments} + if o.base is not None: + # A call through an object, e.g. `obj.method(...)`, where `obj` + # may be in the mapper too + callee = uxreplace(o.callee, self.mapper) + if callee is not o.callee: + kwargs['name'] = callee if o.retobj is not None: - retobj = uxreplace(o.retobj, self.mapper) - return reuse_if_unchanged(o, arguments=arguments, retobj=retobj) - else: - return reuse_if_unchanged(o, arguments=arguments) + kwargs['retobj'] = uxreplace(o.retobj, self.mapper) + return reuse_if_unchanged(o, **kwargs) def visit_Lambda(self, o): body = self._visit(o.body) diff --git a/docker/Dockerfile.intel b/docker/Dockerfile.intel index c7a7a0f071..7d1eca8a01 100644 --- a/docker/Dockerfile.intel +++ b/docker/Dockerfile.intel @@ -37,13 +37,17 @@ RUN apt-get update -y && \ # Drivers mandatory for intel gpu # https://dgpu-docs.intel.com/driver/installation.html#ubuntu-install-steps +# NOTE: The Data Center GPU Max (PVC) user-space stack comes from the LTS channels. +# The legacy `graphics/ubuntu jammy unified` channel stops at compute-runtime 23.17, +# with which the oneAPI 2026 compilers cannot drive a Max 1100 (SYCL finds no Level +# Zero device and OpenMP offload images fail to load) SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN wget -qO - https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor > /usr/share/keyrings/intel-graphics.gpg -RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy unified" > /etc/apt/sources.list.d/intel-gpu-jammy.list +RUN wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --dearmor > /usr/share/keyrings/intel-graphics.gpg +RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy/lts/2523 unified" > /etc/apt/sources.list.d/intel-gpu-jammy.list RUN apt-get update -y && apt-get dist-upgrade -y && \ # Compute and media runtime - apt-get install -y intel-opencl-icd intel-level-zero-gpu level-zero \ + apt-get install -y intel-opencl-icd libze-intel-gpu1 libze1 \ intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \ libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \ libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \ @@ -51,10 +55,13 @@ RUN apt-get update -y && apt-get dist-upgrade -y && \ # XPU-SMI pciutils libkmod-dev intel-fw-gpu xpu-smi \ # Development packages - libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev level-zero-dev + libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev libze-dev ENV MPI4PY_FLAGS='. /opt/intel/oneapi/setvars.sh intel64 && ' ENV MPI4PY_RC_RECV_MPROBE=0 +# Intel MPI 2021.19 picks the `shm` fabric alone on a single node, which +# refuses `MPI_Comm_spawn` (as used by the DevitoPRO decoupler) +ENV I_MPI_FABRICS="shm:ofi" RUN apt-get clean && apt-get autoclean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* @@ -104,6 +111,14 @@ ENV DEVITO_ARCH="sycl" ENV DEVITO_LANGUAGE="sycl" ENV DEVITO_PLATFORM="intel64" +############################################################## +# ICX OpenMP GPU image +############################################################## +FROM icx AS gpu-icx + +# Devito config +ENV DEVITO_PLATFORM="intelgpuX" + ############################################################## # ICX SYCL GPU image ############################################################## diff --git a/tests/test_gpu_common.py b/tests/test_gpu_common.py index c28cc3e61e..b97200bc2d 100644 --- a/tests/test_gpu_common.py +++ b/tests/test_gpu_common.py @@ -35,6 +35,18 @@ def test_get_gpu_info(self): # as we're not really exploiting this info yet... pytest.xfail("Unsupported platform for get_gpu_info") + def test_memavail(self): + """ + The device memory budget of an Operator (e.g., for snapshotting) + needs the available memory of the target device. + """ + plat = configuration['platform'] + + memavail = plat.memavail() + + assert isinstance(memavail, int) + assert 0 < memavail <= plat.memtotal + def custom_compiler(self): grid = Grid(shape=(4, 4)) @@ -84,7 +96,8 @@ class TestDeviceID: 'CUDA_VISIBLE_DEVICES', 'NVIDIA_VISIBLE_DEVICES', 'ROCR_VISIBLE_DEVICES', - 'HIP_VISIBLE_DEVICES' + 'HIP_VISIBLE_DEVICES', + 'ZE_AFFINITY_MASK' ) @pytest.mark.parametrize('env_variables', [{"CUDA_VISIBLE_DEVICES": "1"}, @@ -93,7 +106,8 @@ class TestDeviceID: {"NVIDIA_VISIBLE_DEVICES": "1"}, {"NVIDIA_VISIBLE_DEVICES": "1,2"}, {"ROCR_VISIBLE_DEVICES": "1"}, - {"HIP_VISIBLE_DEVICES": " 1"}]) + {"HIP_VISIBLE_DEVICES": " 1"}, + {"ZE_AFFINITY_MASK": "1"}]) def test_visible_devices(self, env_variables): """ Test that physical device IDs used for querying memory on a device via diff --git a/tests/test_iet.py b/tests/test_iet.py index 99a94ed24c..a5641946b4 100644 --- a/tests/test_iet.py +++ b/tests/test_iet.py @@ -19,8 +19,8 @@ from devito.passes.iet.engine import Graph, reuse_efuncs from devito.passes.iet.languages.C import CDataManager from devito.symbolics import ( - FLOAT, Byref, Class, FieldFromComposite, InlineIf, ListInitializer, Macro, SizeOf, - String + FLOAT, Byref, Class, FieldFromComposite, FieldFromPointer, InlineIf, ListInitializer, + Macro, SizeOf, String ) from devito.symbolics.extended_dtypes import c_complex from devito.tools import CustomDtype, as_tuple, dtype_to_ctype @@ -564,6 +564,40 @@ def make_foo(name, start): assert [i.name for i in calls] == ['foo0', 'foo0'] +def test_reuse_efuncs_call_through_object(): + """ + Callables that differ only in the object a method is called through, + as in `q0->wait()` vs `q1->wait()`, must map to the same efunc. + + No OSS backend emits such Calls, hence an IET-level test. Downstream, the + SYCL backend emits `queue.memcpy(...)` in its streaming threads, and two + streamed Functions used to get two otherwise identical thread functions. + """ + + def make_foo(name, q): + p = Pointer(name=f'p{q.name}') + return make_callable(name, List(body=[ + Call(FieldFromPointer('copy', q), [p]), + Call(FieldFromPointer('wait', q)) + ])) + + foo0 = make_foo('foo0', Pointer(name='q0')) + foo1 = make_foo('foo1', Pointer(name='q1')) + + root = make_callable('root', List(body=[ + Call(foo0.name, foo0.parameters), + Call(foo1.name, foo1.parameters) + ])) + efuncs = {i.name: i for i in (root, foo0, foo1)} + + efuncs = reuse_efuncs(root, efuncs) + + assert set(efuncs) == {'root', 'foo0'} + calls = FindNodes(Call).visit(efuncs[root.name]) + assert [i.name for i in calls] == ['foo0', 'foo0'] + assert [str(i.arguments[1]) for i in calls] == ['q0', 'q1'] + + def test_sorted_efuncs_natural_numbering(): names = ['kernel3', 'foo0', 'kernel11', 'kernel1', 'kernel2'] efuncs = [make_callable(name, List()) for name in names]