From 356284d5244823278c580308224594fb34acb0c1 Mon Sep 17 00:00:00 2001 From: Sandeep Datta <128171450+sandeepd-nv@users.noreply.github.com> Date: Sat, 21 Oct 2023 22:48:20 +0530 Subject: [PATCH] GH build releasables (#805) --- .github/workflows/ci-gh-build-release.yml | 23 ++++++++ .../workflows/ci-gh-cpu-build-and-test.yml | 4 +- .../workflows/ci-gh-gpu-build-and-test.yml | 4 +- .github/workflows/gh-build.yml | 27 +++++---- conda/conda-build/build.sh | 14 +++++ conda/conda-build/conda_build_config.yaml | 4 ++ conda/conda-build/meta.yaml | 57 ++++++++++++++----- continuous_integration/Dockerfile | 14 ++++- continuous_integration/build-docker-image | 55 ++++++++++++------ .../home/coder/.local/bin/build-legate | 35 ++++++++++++ .../home/coder/.local/bin/build-legate-all | 17 ------ .../home/coder/.local/bin/build-legate-conda | 2 + .../home/coder/.local/bin/conda-utils | 5 ++ .../home/coder/.local/bin/copy-artifacts | 23 +++++++- .../home/coder/.local/bin/make-conda-env | 17 ++++++ 15 files changed, 232 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/ci-gh-build-release.yml create mode 100755 continuous_integration/home/coder/.local/bin/build-legate delete mode 100755 continuous_integration/home/coder/.local/bin/build-legate-all create mode 100755 continuous_integration/home/coder/.local/bin/make-conda-env diff --git a/.github/workflows/ci-gh-build-release.yml b/.github/workflows/ci-gh-build-release.yml new file mode 100644 index 000000000..9fde39ecf --- /dev/null +++ b/.github/workflows/ci-gh-build-release.yml @@ -0,0 +1,23 @@ +name: Build conda release packages + +concurrency: + group: ci-release-on-${{ github.event_name }}-from-${{ github.ref_name }} + cancel-in-progress: true + +on: + push: + workflow_dispatch: + branches: + - "pull-request/[0-9]+" + - "branch-*" + +jobs: + build: + uses: + ./.github/workflows/gh-build.yml + with: + build-target: all + repos-name: ${{ github.event.repository.name }} + runs-on: ${{ github.repository_owner == 'nv-legate' && 'linux-amd64-32cpu' || 'ubuntu-latest' }} + sha: ${{ github.sha }} + build-type: release diff --git a/.github/workflows/ci-gh-cpu-build-and-test.yml b/.github/workflows/ci-gh-cpu-build-and-test.yml index c0a49bae6..7ee174639 100644 --- a/.github/workflows/ci-gh-cpu-build-and-test.yml +++ b/.github/workflows/ci-gh-cpu-build-and-test.yml @@ -17,10 +17,10 @@ jobs: ./.github/workflows/gh-build.yml with: build-target: cpu - # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` repos-name: ${{ github.event.repository.name }} runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} + build-type: ci test-cpu: needs: @@ -38,7 +38,7 @@ jobs: with: build-target: cpu repos-name: ${{ github.event.repository.name }} - runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-cpu4' || 'ubuntu-latest' }} + runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-32cpu' || 'ubuntu-latest' }} sha: ${{ github.sha }} test-scope: ${{ matrix.test-scope }} diff --git a/.github/workflows/ci-gh-gpu-build-and-test.yml b/.github/workflows/ci-gh-gpu-build-and-test.yml index abb6d7a15..b0c32f1de 100644 --- a/.github/workflows/ci-gh-gpu-build-and-test.yml +++ b/.github/workflows/ci-gh-gpu-build-and-test.yml @@ -17,10 +17,10 @@ jobs: ./.github/workflows/gh-build.yml with: build-target: gpu - # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` repos-name: ${{ github.event.repository.name }} - runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-cpu4' || 'ubuntu-latest' }} + runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-32cpu' || 'ubuntu-latest' }} sha: ${{ github.sha }} + build-type: ci test-gpu: needs: diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index ea18fea6f..d4a0e216f 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -6,6 +6,7 @@ on: build-target: required: true type: string + description: One of CPU / GPU repos-name: required: true type: string @@ -15,12 +16,18 @@ on: sha: required: true type: string + description: A unique identifier for labeling the images / artifacts + build-type: + required: true + type: string + description: One of ci / release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-mambaforge-ubuntu22.04 IMAGE_NAME: ${{ inputs.repos-name }}-${{ inputs.build-target }} USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} + PUSH_IMAGE: ${{ inputs.build-type == 'ci' && 'true' || 'false'}} jobs: build: @@ -54,29 +61,26 @@ jobs: - name: Build docker image run: | - echo BUILD_TARGET: ${{ inputs.build-target }} - echo USE_CUDA: ${{ env.USE_CUDA }} - IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }} continuous_integration/build-docker-image \ --base-image "$BASE_IMAGE" \ --image-tag "$IMAGE_TAG" \ - --source-dir . + --use-cuda "$USE_CUDA" \ + --source-dir . \ + --build-type "${{ inputs.build-type }}" - name: Dump docker history of image before upload run: | IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }} docker history $IMAGE_TAG - - name: Dump docker history of image before upload - run: | - IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }} - - - name: Log in to container image registry + - if: env.PUSH_IMAGE == 'true' + name: Log in to container image registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - - name: Push image + - if: env.PUSH_IMAGE == 'true' + name: Push image run: | IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }} @@ -94,8 +98,7 @@ jobs: run: | IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }} mkdir -p artifacts - docker run -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts - + docker run -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts ${{ inputs.build-type }} - name: Display structure of workdir run: ls -R diff --git a/conda/conda-build/build.sh b/conda/conda-build/build.sh index 768497bea..73ebbc313 100644 --- a/conda/conda-build/build.sh +++ b/conda/conda-build/build.sh @@ -1,5 +1,9 @@ #!/bin/bash +echo -e "\n\n--------------------- CONDA/CONDA-BUILD/BUILD.SH -----------------------\n" + +set -xeo pipefail + # Rewrite conda's -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY to # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH CMAKE_ARGS="$(echo "$CMAKE_ARGS" | sed -r "s@_INCLUDE=ONLY@_INCLUDE=BOTH@g")" @@ -20,11 +24,21 @@ if [ -z "$CPU_ONLY" ]; then " fi +# We rely on an environment variable to determine if we need to make a debug build. +if [ -n "$DEBUG_BUILD" ]; then + CMAKE_ARGS+=" +-DCMAKE_BUILD_TYPE=Debug +" +fi + # Do not compile with NDEBUG until Legion handles it without warnings +# Note: -UNDEBUG undefines any NDEBUG that may be present on the C compiler commandline. +# See: https://stackoverflow.com/questions/1978155/how-to-undefine-a-define-at-commandline-using-gcc export CFLAGS="-UNDEBUG" export CXXFLAGS="-UNDEBUG" export CPPFLAGS="-UNDEBUG" export CUDAFLAGS="-UNDEBUG" + export CMAKE_GENERATOR=Ninja export CUDAHOSTCXX=${CXX} export OPENSSL_DIR="$CONDA_PREFIX" diff --git a/conda/conda-build/conda_build_config.yaml b/conda/conda-build/conda_build_config.yaml index a508a6ed1..ce3e7e561 100644 --- a/conda/conda-build/conda_build_config.yaml +++ b/conda/conda-build/conda_build_config.yaml @@ -1,3 +1,7 @@ +debug_build: + - true + - false + gpu_enabled: - true - false diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index 65068414b..6edab7b99 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -7,6 +7,14 @@ {# We need to have a default value for the initial pass over the recipe #} {% set gpu_enabled_bool = false %} {% endif %} +{% if debug_build == "true" %} + {% set debug_build_bool = true %} +{% elif debug_build == "false" %} + {% set debug_build_bool = false %} +{% else %} + {# We need to have a default value for the initial pass over the recipe #} + {% set debug_build_bool = false %} +{% endif %} {% set default_env_var = '' %} {% if build_number is defined %} {# do nothing if defined #} @@ -27,6 +35,24 @@ {% set cuda_major=cuda_version.split('.')[0]|int %} {% set py_version=environ.get('CONDA_PY', 36) %} +{% if use_local_path is not defined %} +{% set git_describe_hash='_' + environ.get('GIT_DESCRIBE_HASH', '0')%} +{% else %} +{% set git_describe_hash=''%} +{% endif %} + +{% if not gpu_enabled_bool %} +{% set cpu_tag='_cpu' %} +{% else %} +{% set cpu_tag='' %} +{% endif %} + +{% if debug_build_bool %} +{% set debug_tag='_debug' %} +{% else %} +{% set debug_tag='' %} +{% endif %} + package: name: {{ name|lower }} version: {{ version }} @@ -45,21 +71,9 @@ build: number: {{ build_number }} missing_dso_whitelist: - '*libcuda.so*' -{% if use_local_path is not defined %} -# use git hash -{% if not gpu_enabled_bool %} - string: "cuda{{ cuda_major }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ PKG_BUILDNUM }}_cpu" -{% else %} - string: "cuda{{ cuda_major }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ PKG_BUILDNUM }}" -{% endif %} -{% else %} -# do not use git hash -{% if not gpu_enabled_bool %} - string: "cuda{{ cuda_major }}_py{{ py_version }}_{{ PKG_BUILDNUM }}_cpu" -{% else %} - string: "cuda{{ cuda_major }}_py{{ py_version }}_{{ PKG_BUILDNUM }}" -{% endif %} -{% endif %} + + string: "cuda{{ cuda_major }}_py{{ py_version }}{{ git_describe_hash }}_{{ PKG_BUILDNUM }}{{ cpu_tag }}{{ debug_tag }}" + script_env: - SCCACHE_BUCKET - SCCACHE_REGION @@ -75,6 +89,9 @@ build: - CPU_ONLY=1 track_features: - cpu_only +{% endif %} +{% if debug_build_bool %} + - DEBUG_BUILD=1 {% endif %} run_exports: {% if not gpu_enabled_bool %} @@ -91,6 +108,7 @@ build: {% endif %} requirements: + # Compilers and build system build: - make - rust @@ -98,6 +116,8 @@ requirements: - cmake {{ cmake_version }} - {{ compiler('c') }} =11.2 - {{ compiler('cxx') }} =11.2 + + # Libraries and header files (C/C++). host: - zlib - python @@ -112,10 +132,17 @@ requirements: - cuda-nvtx ={{ cuda_version }} - cuda-cccl ={{ cuda_version }} - cuda-cudart ={{ cuda_version }} + - cuda-cudart-static ={{ cuda_version }} - cuda-nvml-dev ={{ cuda_version }} - cuda-driver-dev ={{ cuda_version }} - cuda-cudart-dev ={{ cuda_version }} + - libcublas-dev + - libcufft-dev + - libcurand-dev + - libcusolver-dev {% endif %} + + # Runtime python dependencies run: - cffi - llvm-openmp diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index d34641f2d..f8407bf41 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -3,6 +3,10 @@ FROM ${BASE_IMAGE} as stage0 SHELL ["/bin/bash", "-c"] +RUN wget https://github.com/mozilla/sccache/releases/download/v0.5.4/sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz && \ + tar -xf sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz && \ + sudo mv sccache-v0.5.4-x86_64-unknown-linux-musl/sccache /usr/bin/sccache + ENV PYTHONDONTWRITEBYTECODE=1 ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-east" @@ -19,7 +23,9 @@ ENV USE_CUDA=${USE_CUDA} ENV USE_OPENMP=ON -ENV BUILD_MARCH=nocona +ENV BUILD_MARCH=haswell + +ARG BUILD_TYPE ENV PATH="${PATH}:/home/coder/.local/bin" @@ -43,7 +49,7 @@ FROM stage0 as setup USER coder WORKDIR /home/coder -RUN set -x && . conda-utils && get_yaml_and_make_conda_env +RUN make-conda-env ${BUILD_TYPE} #--------------------------------------------------- FROM setup as build @@ -52,6 +58,8 @@ WORKDIR /home/coder ARG GITHUB_TOKEN ENV GITHUB_TOKEN=${GITHUB_TOKEN} +ARG AWS_REGION +ENV AWS_REGION=${AWS_REGION} ARG AWS_SESSION_TOKEN ENV AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} ARG AWS_ACCESS_KEY_ID @@ -61,7 +69,7 @@ ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} COPY --chown=coder:coder .creds /run/secrets -RUN entrypoint build-legate-all +RUN entrypoint build-legate ${BUILD_TYPE} #--------------------------------------------------- FROM stage0 as final diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image index b106b7fe5..e71c05f0b 100755 --- a/continuous_integration/build-docker-image +++ b/continuous_integration/build-docker-image @@ -3,8 +3,14 @@ set -xuo pipefail #Set the options of the getopt command -format=$(getopt -n "$0" -l "base-image:,image-tag:,source-dir:" -- -- "$@") -if [ $# -lt 4 ]; then +# NOTE: local-build is optional +format=$(getopt -n "$0" -l "base-image:,image-tag:,source-dir:,build-type:,use-cuda:,local-build" -- -- "$@") +if [ $? != 0 ]; then + echo "Getopt failed" + exit 1 +fi + +if [ $# -lt 6 ]; then echo "Wrong number of arguments passed." exit fi @@ -14,10 +20,14 @@ eval set -- "$format" while [ $# -gt 0 ] do case "$1" in - --base-image) BASE_IMAGE="$2"; shift;; - --image-tag) IMAGE_TAG="$2"; shift;; - --source-dir) SOURCE_DIR="$2"; shift;; - --) shift;; + --base-image) BASE_IMAGE="$2"; shift;; + --image-tag) IMAGE_TAG="$2"; shift;; + --source-dir) SOURCE_DIR="$2"; shift;; + --build-type) BUILD_TYPE="$2"; shift;; + --use-cuda) USE_CUDA="$2"; shift;; + --local-build) LOCAL_BUILD="1";; + --) shift; break;; + *) echo "Invalid option: $1"; exit 1;; esac shift; done @@ -29,13 +39,26 @@ mkdir -p "$SOURCE_DIR/.creds" export DOCKER_BUILDKIT=1 -docker build \ - --build-arg BASE_IMAGE="$BASE_IMAGE" \ - --build-arg AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ - --build-arg AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - --build-arg AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - --build-arg GITHUB_TOKEN="$GITHUB_TOKEN" \ - --build-arg USE_CUDA="$USE_CUDA" \ - --progress=plain \ - --tag="$IMAGE_TAG" \ - -f "$SOURCE_DIR/continuous_integration/Dockerfile" "$SOURCE_DIR" +if [[ -z "${LOCAL_BUILD+x}" ]]; then + docker build \ + --build-arg BASE_IMAGE="$BASE_IMAGE" \ + --build-arg AWS_REGION="$AWS_REGION" \ + --build-arg AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ + --build-arg AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + --build-arg AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + --build-arg GITHUB_TOKEN="$GITHUB_TOKEN" \ + --build-arg USE_CUDA="$USE_CUDA" \ + --build-arg BUILD_TYPE="$BUILD_TYPE" \ + --progress=plain \ + --tag="$IMAGE_TAG" \ + -f "$SOURCE_DIR/continuous_integration/Dockerfile" "$SOURCE_DIR"; +else + docker build \ + --target setup \ + --build-arg BASE_IMAGE="$BASE_IMAGE" \ + --build-arg USE_CUDA="$USE_CUDA" \ + --build-arg BUILD_TYPE="$BUILD_TYPE" \ + --progress=plain \ + --tag="$IMAGE_TAG" \ + -f "$SOURCE_DIR/continuous_integration/Dockerfile" "$SOURCE_DIR"; +fi diff --git a/continuous_integration/home/coder/.local/bin/build-legate b/continuous_integration/home/coder/.local/bin/build-legate new file mode 100755 index 000000000..f2dac14cf --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/build-legate @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + + +build_legate_all() { + build-legate-cpp; + build-legate-wheel; + build-legate-conda; +} + +build_legate_release() { + # export CONDA_BUILD_THREADS=1 + # export NVCC_PREPEND_FLAGS='--verbose' + . conda-utils + activate_conda_env + mkdir -p /tmp/env_yaml /tmp/conda-build + cd legate + conda mambabuild --output-folder /tmp/conda-build -c nvidia -c conda-forge --no-include-recipe conda/conda-build +} + +build_legate() { + set -x + cd ~/; + + conda info + + set -eo pipefail; + + case "$1" in + ci) build_legate_all;; + release) build_legate_release;; + *) return 1;; + esac +} + +(build_legate "$@"); diff --git a/continuous_integration/home/coder/.local/bin/build-legate-all b/continuous_integration/home/coder/.local/bin/build-legate-all deleted file mode 100755 index 46726e1ba..000000000 --- a/continuous_integration/home/coder/.local/bin/build-legate-all +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - - -build_legate_all() { - set -x - cd ~/; - - conda info - - set -euo pipefail; - - build-legate-cpp; - build-legate-wheel; - build-legate-conda; -} - -(build_legate_all "$@"); diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index 2ec7d0db6..8d0e647f2 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -49,6 +49,8 @@ use_local_path: - "true" gpu_enabled: - "${GPU_ENABLED}" +debug_build: + - "false" cmake_version: - ">=3.20.1,!=3.23.0" package_version: diff --git a/continuous_integration/home/coder/.local/bin/conda-utils b/continuous_integration/home/coder/.local/bin/conda-utils index bd5dc38ea..ee6b29bf8 100755 --- a/continuous_integration/home/coder/.local/bin/conda-utils +++ b/continuous_integration/home/coder/.local/bin/conda-utils @@ -75,3 +75,8 @@ activate_conda_env() { conda activate ${DEFAULT_CONDA_ENV:-legate}; } + +make_release_env() { + mamba create -n "${DEFAULT_CONDA_ENV:-legate}" + mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" boa +} \ No newline at end of file diff --git a/continuous_integration/home/coder/.local/bin/copy-artifacts b/continuous_integration/home/coder/.local/bin/copy-artifacts index 9ca293535..95312c1e3 100755 --- a/continuous_integration/home/coder/.local/bin/copy-artifacts +++ b/continuous_integration/home/coder/.local/bin/copy-artifacts @@ -1,10 +1,29 @@ #!/usr/bin/env bash +copy_release_artifacts() { + echo Copying release artifacts + cp /tmp/conda-build/linux-64/legate-core-*.tar.bz2 /home/coder/.artifacts +} + +copy_ci_artifacts() { + echo Copying CI artifacts + + cp -r /tmp/out ~/.artifacts + cp -r /tmp/conda-build ~/.artifacts +} + copy_artifacts() { set -xeuo pipefail + sudo chown coder:coder ~/.artifacts - cp -r /tmp/out ~/.artifacts - cp -r /tmp/conda-build ~/.artifacts + + case "$1" in + ci) copy_ci_artifacts;; + release) copy_release_artifacts;; + *) return 1;; + esac + + return 0; } (copy_artifacts "$@"); diff --git a/continuous_integration/home/coder/.local/bin/make-conda-env b/continuous_integration/home/coder/.local/bin/make-conda-env new file mode 100755 index 000000000..cfcffe3b8 --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/make-conda-env @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +. conda-utils + +make_conda_env() { + set -xeuo pipefail + + case "$1" in + ci) get_yaml_and_make_conda_env;; + release) make_release_env;; + *) return 1;; + esac + + return 0; +} + +(make_conda_env "$@"); \ No newline at end of file