Merge branch 'autonomys:main' into gpu-change-endianness

autonomys · Oct 8, 2024 · d41efdb · d41efdb
2 parents b2e0cfe + 5dce9ce
commit d41efdb
Show file tree

Hide file tree

Showing 30 changed files with 982 additions and 329 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -54,7 +54,7 @@ jobs:
   cargo-clippy:
     strategy:
       matrix:
-        os: ${{ fromJson(github.repository_owner == 'autonomys' && '[["self-hosted", "ubuntu-20.04-x86-64"], ["self-hosted", "macos-14-arm64"], ["self-hosted", "windows-server-2022-x86-64"]]' || '["ubuntu-22.04", "macos-14", "windows-2022"]') }}
+        os: ${{ fromJson(github.repository_owner == 'autonomys' && '[["self-hosted", "ubuntu-20.04-x86-64"], ["self-hosted", "macos-14-arm64"], ["self-hosted", "windows-server-2022-x86-64"]]' || '["ubuntu-20.04", "macos-14", "windows-2022"]') }}
 
     runs-on: ${{ matrix.os }}
 
@@ -104,6 +104,42 @@ jobs:
           sub-packages: '["nvcc", "cudart"]'
         if: runner.os == 'Linux' || runner.os == 'Windows'
 
+      # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it
+      # - name: Configure ROCm cache (Windows)
+      #   uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+      #   id: rocm-cache
+      #   with:
+      #     path: C:\Program Files\AMD\ROCm
+      #     key: ${{ runner.os }}-rocm
+      #   if: runner.os == 'Windows'
+
+      - name: ROCm toolchain
+        run: |
+          ROCM_VERSION=6.2.2
+          sudo mkdir -p --mode=0755 /etc/apt/keyrings
+          curl -L https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+          echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION focal main" | sudo tee /etc/apt/sources.list.d/rocm.list > /dev/null
+          echo "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" | sudo tee /etc/apt/preferences.d/rocm-pin-600 > /dev/null
+          sudo apt-get update
+          DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends rocm-hip-runtime-dev
+          echo "/opt/rocm/lib" | sudo tee /etc/ld.so.conf.d/rocm.conf > /dev/null
+          sudo ldconfig
+        if: runner.os == 'Linux'
+
+      # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it
+      # - name: ROCm toolchain
+      #   run: |
+      #     $ErrorActionPreference = "Stop"
+      #     Invoke-WebRequest -Uri https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe -OutFile "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe"
+      #     Start-Process "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" -ArgumentList '-install' -NoNewWindow -Wait
+      #     Remove-Item "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe"
+      #   if: runner.os == 'Windows' && steps.rocm-cache.outputs.cache-hit != 'true'
+      #
+      # - name: ROCm toolchain environment (Windows)
+      #   run: |
+      #     Add-Content $env:GITHUB_PATH "C:\Program Files\AMD\ROCm\6.1\bin"
+      #   if: runner.os == 'Windows'
+
       - name: Configure cache
         uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
         with:
@@ -124,6 +160,22 @@ jobs:
           cargo -Zgitoxide -Zgit clippy --locked --all-targets --features runtime-benchmarks,cuda -- -D warnings
         if: runner.os == 'Linux' || runner.os == 'Windows'
 
+      - name: cargo clippy (ROCm)
+        env:
+          NVCC: off
+        run: |
+          cargo -Zgitoxide -Zgit clippy --locked --all-targets --features rocm -- -D warnings
+        if: runner.os == 'Linux'
+
+      # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it
+      # - name: cargo clippy (ROCm)
+      #   env:
+      #     NVCC: off
+      #     HIPCC: hipcc.bin.exe
+      #   run: |
+      #     cargo -Zgitoxide -Zgit clippy --locked --all-targets --features rocm -- -D warnings
+      #   if: runner.os == 'Windows'
+
   cargo-docs:
     runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }}
     steps:

diff --git a/.github/workflows/snapshot-build.yml b/.github/workflows/snapshot-build.yml
@@ -18,7 +18,9 @@ env:
 
 jobs:
   container-linux:
-    runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }}
+    # TODO: Restore once we have self-hosted runners with Ubuntu 22.04
+    #runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }}
+    runs-on: ubuntu-22.04
     permissions:
       contents: write
       packages: write
@@ -28,28 +30,17 @@ jobs:
           - farmer
           - node
           - bootstrap-node
-        platform:
-          - arch: linux/amd64
-            dockerfile-suffix: ""
-            suffix: ubuntu-x86_64-${{ github.ref_name }}
-            image-suffix: ""
-            rustflags: "-C target-cpu=skylake"
-          # We build AArch64
-          - arch: linux/amd64
-            dockerfile-suffix: ".aarch64"
-            suffix: ubuntu-aarch64-${{ github.ref_name }}
-            image-suffix: "-aarch64"
       fail-fast: false
 
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
+        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
 
       - name: Log into registry
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -67,19 +58,18 @@ jobs:
             type=sha,format=long
           flavor: |
             latest=false
-            suffix=${{ matrix.platform.image-suffix }}
+            suffix=
 
       - name: Build and push ${{ matrix.image }} image
-        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
         with:
-          file: Dockerfile-${{ matrix.image }}${{ matrix.platform.dockerfile-suffix }}
-          platforms: ${{ matrix.platform.arch }}
+          file: Dockerfile-${{ matrix.image }}
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             SUBSTRATE_CLI_GIT_COMMIT_HASH=${{ github.sha }}
-            RUSTFLAGS=${{ matrix.platform.rustflags }}
 
       - name: Trigger trivy-security-scan Workflow
         uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # @v3.0.0
@@ -178,6 +168,43 @@ jobs:
           sub-packages: '["nvcc", "cudart"]'
         if: runner.os == 'Linux' || runner.os == 'Windows'
 
+      # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it
+      # - name: Configure ROCm cache (Windows)
+      #   uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+      #   id: rocm-cache
+      #   with:
+      #     path: C:\Program Files\AMD\ROCm
+      #     key: ${{ runner.os }}-rocm
+      #   if: runner.os == 'Windows'
+
+      - name: ROCm toolchain
+        run: |
+          ROCM_VERSION=6.2.2
+          sudo mkdir -p --mode=0755 /etc/apt/keyrings
+          curl -L https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+          echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION focal main" | sudo tee /etc/apt/sources.list.d/rocm.list > /dev/null
+          echo "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" | sudo tee /etc/apt/preferences.d/rocm-pin-600 > /dev/null
+          sudo apt-get update
+          DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends rocm-hip-runtime-dev
+          echo "/opt/rocm/lib" | sudo tee /etc/ld.so.conf.d/rocm.conf > /dev/null
+          sudo ldconfig
+        # TODO: ROCm packages are only available for x86-64 for now
+        if: runner.os == 'Linux' && startsWith(matrix.build.target, 'x86_64')
+
+      # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it
+      # - name: ROCm toolchain
+      #   run: |
+      #     $ErrorActionPreference = "Stop"
+      #     Invoke-WebRequest -Uri https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe -OutFile "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe"
+      #     Start-Process "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" -ArgumentList '-install' -NoNewWindow -Wait
+      #     Remove-Item "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe"
+      #   if: runner.os == 'Windows' && steps.rocm-cache.outputs.cache-hit != 'true'
+      #
+      # - name: ROCm toolchain environment (Windows)
+      #   run: |
+      #     Add-Content $env:GITHUB_PATH "C:\Program Files\AMD\ROCm\6.1\bin"
+      #   if: runner.os == 'Windows'
+
       - name: Configure cache
         uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
         with:
@@ -193,6 +220,28 @@ jobs:
           cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer
         if: runner.os == 'macOS' || !startsWith(matrix.build.target, 'x86_64')
 
+      # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it
+      # # ROCm can't be enabled together with CUDA for now
+      # - name: Build farmer (ROCm, Windows)
+      #   env:
+      #     NVCC: off
+      #     HIPCC: hipcc.bin.exe
+      #   run: |
+      #     cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features rocm
+      #     move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe
+      #   # TODO: ROCm packages are only available for x86-64 for now
+      #   if: runner.os == 'Windows' && startsWith(matrix.build.target, 'x86_64')
+
+      # ROCm can't be enabled together with CUDA for now
+      - name: Build farmer (ROCm, Ubuntu)
+        env:
+          NVCC: off
+        run: |
+          cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features rocm
+          mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm
+        # TODO: ROCm packages are only available for x86-64 for now
+        if: runner.os == 'Linux' && startsWith(matrix.build.target, 'x86_64')
+
       - name: Build farmer
         run: |
           cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features cuda
@@ -240,6 +289,8 @@ jobs:
       - name: Sign Application (Windows)
         run: |
           AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe"
+          # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it
+          # AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe"
           AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-node.exe"
         # Allow code signing to fail on non-release builds and in non-subspace repos (forks)
         continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }}
@@ -252,6 +303,12 @@ jobs:
           mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }}
         if: runner.os == 'Linux'
 
+      - name: Prepare executables for uploading (Ubuntu, ROCm)
+        run: |
+          mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm executables/subspace-farmer-rocm-${{ matrix.build.suffix }}
+        # TODO: ROCm packages are only available for x86-64 for now
+        if: runner.os == 'Linux' && startsWith(matrix.build.target, 'x86_64')
+
       - name: Prepare executables for uploading (macOS)
         run: |
           mkdir executables
@@ -268,6 +325,8 @@ jobs:
         run: |
           mkdir executables
           move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe executables/subspace-farmer-${{ matrix.build.suffix }}.exe
+          # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it
+          # move ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe executables/subspace-farmer-rocm-${{ matrix.build.suffix }}.exe
           move ${{ env.PRODUCTION_TARGET }}/subspace-node.exe executables/subspace-node-${{ matrix.build.suffix }}.exe
         if: runner.os == 'Windows'
 

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Dockerfile-bootstrap-node b/Dockerfile-bootstrap-node
@@ -1,12 +1,15 @@
-FROM ubuntu:20.04
+# This Dockerfile supports both native building and cross-compilation to aarch64 on x86-64
+FROM --platform=$BUILDPLATFORM ubuntu:22.04
 
 ARG RUSTC_VERSION=nightly-2024-09-26
 ARG PROFILE=production
 ARG RUSTFLAGS
-# Workaround for https://github.com/rust-lang/cargo/issues/10583
-ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
 # Incremental compilation here isn't helpful
 ENV CARGO_INCREMENTAL=0
+ENV PKG_CONFIG_ALLOW_CROSS=true
+
+ARG BUILDARCH
+ARG TARGETARCH
 
 WORKDIR /code
 
@@ -22,10 +25,19 @@ RUN \
         automake \
         libtool \
         pkg-config \
-        make && \
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION
+        make
+
+RUN \
+    if [ $BUILDARCH != "arm64" ] && [ $TARGETARCH = "arm64" ]; then \
+      DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+          g++-aarch64-linux-gnu \
+          gcc-aarch64-linux-gnu \
+          libc6-dev-arm64-cross \
+    ; fi
 
-RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown
+RUN \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain $RUSTC_VERSION && \
+    /root/.cargo/bin/rustup target add wasm32-unknown-unknown
 
 COPY Cargo.lock /code/Cargo.lock
 COPY Cargo.toml /code/Cargo.toml
@@ -39,16 +51,23 @@ COPY test /code/test
 # Up until this line all Rust images in this repo should be the same to share the same layers
 
 RUN \
+    if [ $BUILDARCH != "arm64" ] && [ $TARGETARCH = "arm64" ]; then \
+      export RUSTFLAGS="$RUSTFLAGS -C linker=aarch64-linux-gnu-gcc" \
+    ; fi && \
+    if [ $TARGETARCH = "amd64" ] && [ $RUSTFLAGS = ""]; then \
+      export RUSTFLAGS="-C target-cpu=skylake" \
+    ; fi && \
+    RUSTC_TARGET_ARCH=$(echo $TARGETARCH | sed "s/amd64/x86_64/g" | sed "s/arm64/aarch64/g") && \
     /root/.cargo/bin/cargo -Zgitoxide -Zgit build \
         --locked \
         -Z build-std \
         --profile $PROFILE \
         --bin subspace-bootstrap-node \
-        --target $(uname -p)-unknown-linux-gnu && \
+        --target $RUSTC_TARGET_ARCH-unknown-linux-gnu && \
     mv target/*/*/subspace-bootstrap-node subspace-bootstrap-node && \
     rm -rf target
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 COPY --from=0 /code/subspace-bootstrap-node /subspace-bootstrap-node