diff --git a/.github/workflows/docker-bases.yml b/.github/workflows/docker-bases.yml index d2bed47180..96846320fc 100644 --- a/.github/workflows/docker-bases.yml +++ b/.github/workflows/docker-bases.yml @@ -162,47 +162,6 @@ jobs: build-args: 'arch=nvc-host' tags: 'devitocodes/bases:cpu-nvc' -####################################################### -################### Nvidia clang ###################### -####################################################### - deploy-nvidia-clang-base: - name: "nvidia-clang-base" - runs-on: ["self-hosted", "nvidiagpu"] - env: - DOCKER_BUILDKIT: "1" - - steps: - - name: Checkout devito - uses: actions/checkout@v3 - - - name: Check event name - run: echo ${{ github.event_name }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: cleanup - run: docker system prune -a -f - - - name: Nvidia clang image - uses: docker/build-push-action@v3 - with: - context: . - file: './docker/Dockerfile.nvidia' - push: true - target: 'clang' - build-args: 'arch=clang' - tags: 'devitocodes/bases:nvidia-clang' - ####################################################### ##################### AMD ############################# ####################################################### @@ -252,4 +211,4 @@ jobs: target: 'hip' build-args: | arch=hip - tags: devitocodes/bases:amd-hip \ No newline at end of file + tags: devitocodes/bases:amd-hip diff --git a/.github/workflows/docker-devito.yml b/.github/workflows/docker-devito.yml index 459093ed6e..597eed6740 100644 --- a/.github/workflows/docker-devito.yml +++ b/.github/workflows/docker-devito.yml @@ -24,12 +24,6 @@ jobs: test: 'tests/test_gpu_openacc.py tests/test_gpu_common.py' runner: ["self-hosted", "nvidiagpu"] - - base: 'bases:nvidia-clang' - tag: 'nvidia-clang' - flag: '--gpus all' - test: 'tests/test_gpu_openmp.py tests/test_gpu_common.py' - runner: ["self-hosted", "nvidiagpu"] - # Runtime gpu flags from https://hub.docker.com/r/rocm/tensorflow/ - base: 'bases:amd' tag: 'amd' diff --git a/.github/workflows/pytest-gpu.yml b/.github/workflows/pytest-gpu.yml index e52134add2..71f0033106 100644 --- a/.github/workflows/pytest-gpu.yml +++ b/.github/workflows/pytest-gpu.yml @@ -46,20 +46,12 @@ jobs: matrix: name: [ - pytest-gpu-omp-nvidia, pytest-gpu-acc-nvidia, pytest-gpu-omp-amd ] test_examples: ["examples/seismic/tti/tti_example.py examples/seismic/acoustic/acoustic_example.py examples/seismic/viscoacoustic/viscoacoustic_example.py examples/seismic/viscoelastic/viscoelastic_example.py examples/seismic/elastic/elastic_example.py"] include: - - name: pytest-gpu-omp-nvidia - test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py" - base: "devitocodes/bases:nvidia-clang" - tags: ["self-hosted", "nvidiagpu"] - test_drive_cmd: "nvidia-smi" - flags: '--gpus all --rm --name testrun-clang-nvidia' - - name: pytest-gpu-acc-nvidia test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openacc.py" base: "devitocodes/bases:nvidia-nvc" diff --git a/devito/finite_differences/derivative.py b/devito/finite_differences/derivative.py index 54aa459479..e8db6e882e 100644 --- a/devito/finite_differences/derivative.py +++ b/devito/finite_differences/derivative.py @@ -89,7 +89,7 @@ class Derivative(sympy.Derivative, Differentiable): __rkwargs__ = ('side', 'deriv_order', 'fd_order', 'transpose', '_ppsubs', 'x0') def __new__(cls, expr, *dims, **kwargs): - if type(expr) == sympy.Derivative: + if type(expr) is sympy.Derivative: raise ValueError("Cannot nest sympy.Derivative with devito.Derivative") if not isinstance(expr, Differentiable): raise ValueError("`expr` must be a Differentiable object") diff --git a/devito/ir/iet/nodes.py b/devito/ir/iet/nodes.py index fc9e907d75..989f557a0b 100644 --- a/devito/ir/iet/nodes.py +++ b/devito/ir/iet/nodes.py @@ -185,7 +185,7 @@ class List(Node): def __init__(self, header=None, body=None, footer=None): body = as_tuple(body) - if len(body) == 1 and all(type(i) == List for i in [self, body[0]]): + if len(body) == 1 and all(type(i) is List for i in [self, body[0]]): # De-nest Lists # # Note: to avoid disgusting metaclass voodoo (due to diff --git a/devito/ir/support/space.py b/devito/ir/support/space.py index 48833699ad..b85b017b4a 100644 --- a/devito/ir/support/space.py +++ b/devito/ir/support/space.py @@ -36,7 +36,7 @@ def __init__(self, dim, stamp=S0): self.stamp = stamp def __eq__(self, o): - return (type(self) == type(o) and + return (type(self) is type(o) and self.dim is o.dim and self.stamp == o.stamp) diff --git a/devito/passes/clusters/aliases.py b/devito/passes/clusters/aliases.py index 106a0c8c79..1e8626da18 100644 --- a/devito/passes/clusters/aliases.py +++ b/devito/passes/clusters/aliases.py @@ -932,7 +932,7 @@ def pick_best(variants, schedule_strategy, eval_variants_delta): Return the variant with the best trade-off between operation count reduction and working set increase. Heuristics may be applied. """ - if type(schedule_strategy) == int: + if type(schedule_strategy) is int: try: return variants[schedule_strategy] except IndexError: diff --git a/devito/symbolics/inspection.py b/devito/symbolics/inspection.py index aae2e4e414..cc05188cfb 100644 --- a/devito/symbolics/inspection.py +++ b/devito/symbolics/inspection.py @@ -42,7 +42,7 @@ def compare_ops(e1, e2): >>> compare_ops(u[x] + u[x+1], u[x] + u[y+10]) True """ - if type(e1) == type(e2) and len(e1.args) == len(e2.args): + if type(e1) is type(e2) and len(e1.args) == len(e2.args): if e1.is_Atom: return True if e1 == e2 else False elif e1.is_Indexed and e2.is_Indexed: diff --git a/devito/types/dimension.py b/devito/types/dimension.py index 19b0026628..6626f09517 100644 --- a/devito/types/dimension.py +++ b/devito/types/dimension.py @@ -1582,7 +1582,7 @@ def _separate_dims(cls, d0, d1, ofs_items): def dimensions(names): - assert type(names) == str + assert type(names) is str return tuple(Dimension(i) for i in names.split()) diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index 478c702c8e..afa26ceff4 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -72,11 +72,6 @@ ENV PATH $I_MPI_ROOT/libfabric/bin:$I_MPI_ROOT/bin:$I_ICC_ROOT/linux/bin/intel64 ENV LD_LIBRARY_PATH $I_MPI_ROOT/libfabric/lib:$I_MPI_ROOT/lib/release:$I_MPI_ROOT/lib:$I_ICC_ROOT/linux/lib:$I_ICC_ROOT/linux/lib/x64:$I_ICC_ROOT/linux/compiler/lib/intel64_lin:${LD_LIBRARY_PATH} ENV FI_PROVIDER_PATH $I_MPI_ROOT/libfabric/lib/prov:/usr/lib64/libfabric:${LD_LIBRARY_PATH} -# Install intelpython -RUN wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/03aae3a8-623a-47cf-9655-5dd8fcf86430/l_pythoni39_oneapi_p_2023.2.0.49422.sh && \ - chmod +x l_pythoni39_oneapi_p_2023.2.0.49422.sh && ./l_pythoni39_oneapi_p_2023.2.0.49422.sh -r yes -a -s --eula accept -ENV PATH /opt/intel/oneapi/intelpython/python3.9/bin:${PATH} - ############################################################## # ICC image # This is a legacy setup that is not built anymore but kept for reference diff --git a/docker/Dockerfile.devito b/docker/Dockerfile.devito index 089ec5b2f8..9c5578844e 100644 --- a/docker/Dockerfile.devito +++ b/docker/Dockerfile.devito @@ -27,6 +27,16 @@ RUN python3 -m venv /venv && \ /venv/bin/pip install --no-cache-dir -e /app/devito[extras,mpi,tests] && \ rm -rf ~/.cache/pip +# Usefull utilities +# Nvtop +RUN apt update && apt install -y git cmake libncurses5-dev libncursesw5-dev libdrm-dev libsystemd-dev cmake && \ + git clone https://github.com/Syllo/nvtop.git /app/nvtop && \ + mkdir -p /app/nvtop/build && cd /app/nvtop/build && \ + cmake .. -DNVIDIA_SUPPORT=ON -DAMDGPU_SUPPORT=ON -DINTEL_SUPPORT=ON && \ + make && \ + make install && \ + ln -fs /app/nvtop/build/src/nvtop /venv/bin/nvtop + # Safety cleanup RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ rm -rf /var/lib/apt/lists/* @@ -63,6 +73,9 @@ RUN chmod +x /print-defaults /jupyter /tests /docker-entrypoint.sh # Venv COPY --from=builder --chown=app:app /venv /venv +# opt +COPY --from=builder --chown=app:app /opt /opt + # Install codecov WORKDIR /app/devito RUN curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && chown app:app codecov diff --git a/docker/Dockerfile.nvidia b/docker/Dockerfile.nvidia index b2bfb62fde..c34ab32446 100644 --- a/docker/Dockerfile.nvidia +++ b/docker/Dockerfile.nvidia @@ -8,11 +8,16 @@ ARG arch="nvc" ######################################################################## # Build base image with apt setup and common env ######################################################################## -FROM ${pyversion}-slim-bullseye as sdk-base +FROM ubuntu:22.04 as sdk-base ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update -y && apt-get install -y -q gpg apt-utils curl wget vim libnuma-dev tmux numactl +RUN apt-get update -y && \ + apt-get install -y -q gpg apt-utils curl wget vim libnuma-dev tmux numactl + +#Install python +RUN apt-get update && \ + apt-get install -y dh-autoreconf python3-venv python3-dev python3-pip # nodesource: nvdashboard requires nodejs>=10 RUN curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | gpg --yes --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg @@ -81,7 +86,7 @@ RUN export NVARCH=$(ls -1 /opt/nvidia/hpc_sdk/Linux_x86_64/ | grep '\.' | head - ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/comm_libs/${CUDA_V}/nccl /opt/nvhpc/comm_libs/nccl # Starting nvhpc 23.5 and cuda 12.1, hpcx and openmpi are inside the cuda version folder, only the bin is in the comm_libs path -RUN export CUDA_V=$(ls /opt/nvhpc/${NVARCH}/cuda/ | grep '\.') && \ +RUN export CUDA_V=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p') && \ ls /opt/nvhpc/comm_libs/${CUDA_V}/hpcx/ &&\ if [ -d /opt/nvhpc/comm_libs/${CUDA_V}/hpcx ]; then \ rm -rf /opt/nvhpc/comm_libs/hpcx && rm -rf /opt/nvhpc/comm_libs/openmpi4 && \ @@ -167,48 +172,3 @@ FROM nvc as nvc-host ENV DEVITO_ARCH="nvc" ENV DEVITO_PLATFORM="cpu64" ENV DEVITO_LANGUAGE="openmp" - -######################################################################## -# Build latest stable clang. This is following the wiki: -# https://en.wikibooks.org/wiki/LLVM_Compiler/Installation. -######################################################################## -# This will only trigger if arch is clang since the final stage depends on it -FROM sdk-base as clang - -## Install clang requirements -RUN apt-get -y update && apt-get install -y -q libelf-dev libffi-dev cmake git gcc-multilib g++-multilib - -######################################################################## -# Build clang 14 with gcc -######################################################################## -RUN git -c advice.detachedHead=false clone --depth 1 https://github.com/devitocodes/llvm-project.git - -RUN mkdir -p /llvm-project/build -RUN cd /llvm-project/build && \ - cmake ../llvm/ -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD="X86;NVPTX" \ - -DLLVM_ENABLE_PROJECTS="clang" \ - -DLLVM_ENABLE_RUNTIMES="openmp" \ - -DCLANG_OPENMP_NVPTX_DEFAULT_ARCH=sm_86 \ - -DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES=all \ - -DCMAKE_INSTALL_PREFIX=/llvm && \ - make -j ${nproc} && \ - make install - -# Set path -ENV PATH /llvm/bin:${PATH} -ENV LD_LIBRARY_PATH /llvm/lib:${LD_LIBRARY_PATH} -ENV CPATH /llvm/include:${CPATH} - -RUN rm -rf llvm-project -RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ - rm -rf /var/lib/apt/lists/* - -# Recompile mpi4py with clang -ENV OMPI_CC="clang" -RUN /venv/bin/pip install --no-cache-dir mpi4py && rm -rf ~/.cache/pip - -# Devito env vars -ENV DEVITO_ARCH="clang" -ENV DEVITO_PLATFORM="nvidiaX" -ENV DEVITO_LANGUAGE="openmp" diff --git a/docker/README.md b/docker/README.md index 10e8996a5d..dbb79b5a3e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -40,10 +40,9 @@ In addition, the following legacy tags are available: ### [Devito] on GPU -Second, we provide three images to run [Devito] on GPUs, tagged `devito:nvidia-nvc-*`, `devito:nvidia-clang-*`, and `devito:amd-*`. +Second, we provide three images to run [Devito] on GPUs, tagged `devito:nvidia-nvc-*`, and `devito:amd-*`. - `devito:nvidia-nvc-*` is intended to be used on NVidia GPUs. It comes with the configuration to use the `nvc` compiler for `openacc` offloading. This image also comes with CUDA-aware MPI for multi-GPU deployment. -- `devito:nvidia-clang-*` is intended to be used on NVidia GPUs. It comes with the configuration to use the `clang` compiler for `openmp` offloading. This image also comes with CUDA-aware MPI for multi-GPU deployment. - `devito:amd-*` is intended to be used on AMD GPUs. It comes with the configuration to use the `aoompcc` compiler for `openmp` offloading. This image also comes with ROCm-aware MPI for multi-GPU deployment. This image can also be used on AMD CPUs since the ROCm compilers are preinstalled. #### NVidia @@ -93,19 +92,12 @@ To build the GPU image with `openacc` offloading and the `nvc` compiler, run: docker build --build-arg base=devitocodes/bases:nvidia-nvc --network=host --file docker/Dockerfile.devito --tag devito . ``` -or if you wish to use the `clang` compiler with `openmp` offloading: - -```bash -docker build --build-arg base=devitocodes/bases:nvidia-clang --network=host --file docker/Dockerfile --tag devito . -``` - -and finally, for AMD architectures: +or if you wish to use the `amdclang` compiler with `openmp` offloading for AMD architectures: ```bash docker build --build-arg base=devitocodes/bases:amd --network=host --file docker/Dockerfile --tag devito . ``` - ## Debugging a base image To build the base image yourself locally, you need to run the standard build command using the provided Dockerfile. diff --git a/requirements-nvidia.txt b/requirements-nvidia.txt index 5f749fc61c..2e7534ea43 100644 --- a/requirements-nvidia.txt +++ b/requirements-nvidia.txt @@ -1,4 +1,4 @@ -cupy-cuda110 +cupy-cuda12x dask-cuda jupyterlab>=3 jupyterlab-nvdashboard diff --git a/tests/test_builtins.py b/tests/test_builtins.py index 0548039a3d..0c70d63e86 100644 --- a/tests/test_builtins.py +++ b/tests/test_builtins.py @@ -490,7 +490,7 @@ def test_issue_1863(self): assert v0 == v1 assert v0 == v2 assert v0 == v3 - assert type(v0) == np.int16 - assert type(v1) == np.int32 - assert type(v2) == np.float32 - assert type(v3) == np.float64 + assert type(v0) is np.int16 + assert type(v1) is np.int32 + assert type(v2) is np.float32 + assert type(v3) is np.float64 diff --git a/tests/test_derivatives.py b/tests/test_derivatives.py index 3ad27f7040..a303e929ae 100644 --- a/tests/test_derivatives.py +++ b/tests/test_derivatives.py @@ -197,7 +197,7 @@ def test_derivatives_space(self, derivative, dim, order): s_expr = u.diff(dim).as_finite_difference(indices).evalf(_PRECISION) assert(simplify(expr - s_expr) == 0) # Symbolic equality - assert type(expr) == EvalDerivative + assert type(expr) is EvalDerivative expr1 = s_expr.func(*expr.args) assert(expr1 == s_expr) # Exact equality @@ -217,7 +217,7 @@ def test_second_derivatives_space(self, derivative, dim, order): indices = [(dim + i * dim.spacing) for i in range(-width, width + 1)] s_expr = u.diff(dim, dim).as_finite_difference(indices).evalf(_PRECISION) assert(simplify(expr - s_expr) == 0) # Symbolic equality - assert type(expr) == EvalDerivative + assert type(expr) is EvalDerivative expr1 = s_expr.func(*expr.args) assert(expr1 == s_expr) # Exact equality diff --git a/tests/test_dimension.py b/tests/test_dimension.py index 702f931740..408dd3ffb5 100644 --- a/tests/test_dimension.py +++ b/tests/test_dimension.py @@ -443,8 +443,8 @@ def test_subdimmiddle_parallel(self, opt): thickness_left=thickness, thickness_right=thickness) # a 5 point stencil that can be computed in parallel - centre = Eq(u[t+1, xi, yi], u[t, xi, yi] + u[t, xi-1, yi] - + u[t, xi+1, yi] + u[t, xi, yi-1] + u[t, xi, yi+1]) + centre = Eq(u[t+1, xi, yi], u[t, xi, yi] + u[t, xi-1, yi] + + u[t, xi+1, yi] + u[t, xi, yi-1] + u[t, xi, yi+1]) u.data[0, 10, 10] = 1.0