Skip to content

Snapshot build

Snapshot build #389

Workflow file for this run

# This action enables building container images and executables for farmer and node, can be triggered manually or by
# release creation.
#
# Container images are only pushed to GitHub Container Registry for releases.
# Executables are built both for releases and for manually triggered runs, uploaded to artifacts and assets.
name: Snapshot build
on:
workflow_dispatch:
push:
tags:
- "snapshot-*"
- "gemini-*"
# Incremental compilation here isn't helpful
env:
CARGO_INCREMENTAL: 0
jobs:
container-linux:
runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }}
permissions:
contents: write
packages: write
strategy:
matrix:
image:
- farmer
- node
- bootstrap-node
platform:
- arch: linux/amd64
dockerfile-suffix: ""
suffix: ubuntu-x86_64-${{ github.ref_name }}
image-suffix: ""
rustflags: "-C target-cpu=skylake"
# We build AArch64
- arch: linux/amd64
dockerfile-suffix: ".aarch64"
suffix: ubuntu-aarch64-${{ github.ref_name }}
image-suffix: "-aarch64"
fail-fast: false
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
- name: Log into registry
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
with:
images: |
ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
tags: |
type=ref,event=tag
type=ref,event=branch
type=sha,format=long
flavor: |
latest=false
suffix=${{ matrix.platform.image-suffix }}
- name: Build and push ${{ matrix.image }} image
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
file: Dockerfile-${{ matrix.image }}${{ matrix.platform.dockerfile-suffix }}
platforms: ${{ matrix.platform.arch }}
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
SUBSTRATE_CLI_GIT_COMMIT_HASH=${{ github.sha }}
RUSTFLAGS=${{ matrix.platform.rustflags }}
- name: Trigger trivy-security-scan Workflow
uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # @v3.0.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
event-type: trivy-scan-dispatch
client-payload: '{"image": "ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:sha-${{ github.sha }}"}'
executables:
strategy:
matrix:
build:
- os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
target: x86_64-unknown-linux-gnu
suffix: ubuntu-x86_64-skylake-${{ github.ref_name }}
rustflags: "-C target-cpu=skylake"
- os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
target: x86_64-unknown-linux-gnu
suffix: ubuntu-x86_64-v2-${{ github.ref_name }}
rustflags: "-C target-cpu=x86-64-v2"
- os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }}
target: aarch64-unknown-linux-gnu
suffix: ubuntu-aarch64-${{ github.ref_name }}
rustflags: "-C linker=aarch64-linux-gnu-gcc"
- os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "macos-14-arm64"]' || '"macos-14"') }}
target: aarch64-apple-darwin
suffix: macos-aarch64-${{ github.ref_name }}
- os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }}
target: x86_64-pc-windows-msvc
suffix: windows-x86_64-skylake-${{ github.ref_name }}
rustflags: "-C target-cpu=skylake"
- os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }}
target: x86_64-pc-windows-msvc
suffix: windows-x86_64-v2-${{ github.ref_name }}
rustflags: "-C target-cpu=x86-64-v2"
fail-fast: false
runs-on: ${{ matrix.build.os }}
env:
PRODUCTION_TARGET: target/${{ matrix.build.target }}/production
RUSTFLAGS: ${{ matrix.build.rustflags }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
# On macOS, we need a proper Clang version, not Apple's custom version without wasm32 support
- name: Install LLVM and Clang for macOS
uses: KyleMayes/install-llvm-action@dec985c8d7b46a2f363ea1a78f660c946a3349ea # v2.0.1
with:
env: true
version: 17
if: runner.os == 'macOS'
# Because macOS, see https://andreasfertig.blog/2021/02/clang-and-gcc-on-macos-catalina-finding-the-include-paths/
- name: Configure C compiler macOS
run: |
echo "SDKROOT=$(xcrun --show-sdk-path)" >> $GITHUB_ENV
if: runner.os == 'macOS'
- name: Install glibtoolize (macOS)
run: brew install libtool
if: runner.os == 'macOS'
- name: Install Protoc
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Needed for hwloc
- name: Install automake (macOS)
run: brew install automake
if: runner.os == 'macOS'
# Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll
- name: Remove msys64
run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse
# Doesn't exist on self-hosted runners
if: matrix.os == 'windows-2022'
- name: AArch64 cross-compile packages
run: |
FLAVOR="$(lsb_release -sc)"
sudo tee /etc/apt/sources.list.d/arm64.list <<LIST
deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} main restricted
deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates main restricted
deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} universe
deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates universe
deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} multiverse
deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates multiverse
deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-backports main restricted universe multiverse
LIST
sudo sed -i 's/deb http/deb [arch=amd64] http/' /etc/apt/sources.list
# GitHub runners use mirror file
sudo sed -i 's/deb mirror/deb [arch=amd64] mirror/' /etc/apt/sources.list
sudo dpkg --add-architecture arm64
sudo apt-get update
# zlib1g-dev:arm64 is only necessary because amd64 version is present on the host and cross-compilation of
# hwlocality-sys fails otherwise
sudo apt-get install -y --no-install-recommends \
g++-aarch64-linux-gnu \
gcc-aarch64-linux-gnu \
libc6-dev-arm64-cross \
zlib1g-dev:arm64
echo "PKG_CONFIG_ALLOW_CROSS=true" >> $GITHUB_ENV
if: matrix.build.target == 'aarch64-unknown-linux-gnu'
- name: CUDA toolchain
uses: Jimver/cuda-toolkit@9b295696791d75d658d8de64c4a85097ad8abeaf # v0.2.16
with:
cuda: '12.4.1'
method: network
sub-packages: '["nvcc", "cudart"]'
if: runner.os == 'Linux' || runner.os == 'Windows'
- name: Configure cache
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
with:
path: |
~/.cargo/registry
~/.cargo/git
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
restore-keys: |
${{ runner.os }}-cargo-
- name: Build farmer
run: |
cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer
if: runner.os == 'macOS' || !startsWith(matrix.build.target, 'x86_64')
- name: Build farmer
run: |
cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features cuda
# TODO: We don't configure CUDA for cross-compilation purposes, hence only x86-64 for now
if: (runner.os == 'Linux' || runner.os == 'Windows') && startsWith(matrix.build.target, 'x86_64')
- name: Build node
run: |
cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-node
- name: Sign Application (macOS)
run: |
echo "Importing certificate"
echo "${{ secrets.MACOS_CERTIFICATE }}" | base64 --decode > certificate.p12
security create-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain
security default-keychain -s build.keychain
security unlock-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain
security import certificate.p12 -k build.keychain -P "${{ secrets.MACOS_CERTIFICATE_PW }}" -T /usr/bin/codesign
security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain
echo "Signing farmer"
codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-farmer
echo "Signing node"
codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-node
echo "Creating an archive"
mkdir ${{ env.PRODUCTION_TARGET }}/macos-binaries
cp ${{ env.PRODUCTION_TARGET }}/subspace-farmer ${{ env.PRODUCTION_TARGET }}/subspace-node ${{ env.PRODUCTION_TARGET }}/macos-binaries
ditto -c -k --rsrc ${{ env.PRODUCTION_TARGET }}/macos-binaries subspace-binaries.zip
echo "Notarizing"
brew update
brew install mitchellh/gon/gon
cat << EOF > gon.hcl
source = ["subspace-binaries.zip"]
bundle_id = "${{ secrets.MACOS_BUNDLE_ID }}"
sign {
application_identity = "${{ secrets.MACOS_IDENTITY }}"
}
apple_id {
username = "${{ secrets.MACOS_APPLE_ID }}"
password = "${{ secrets.MACOS_APP_PW }}"
}
EOF
gon -log-level=info -log-json gon.hcl
# Notarize the ZIP using notarytool
xcrun notarytool submit subspace-binaries.zip --apple-id "${{ secrets.MACOS_APPLE_ID }}" --password "${{ secrets.MACOS_APP_PW }}" --team-id "${{ secrets.MACOS_TEAM_ID }}" --wait
# // todo stapling for macOS artifacts
# Staple the zip package
# xcrun stapler staple subspace-binaries.zip
echo "Done!"
# Allow code signing to fail on non-release builds and in non-subspace repos (forks)
continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }}
if: runner.os == 'macOS'
- name: Sign Application (Windows)
run: |
AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe"
AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-node.exe"
# Allow code signing to fail on non-release builds and in non-subspace repos (forks)
continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }}
if: runner.os == 'Windows'
- name: Prepare executables for uploading (Ubuntu)
run: |
mkdir executables
mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }}
mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }}
if: runner.os == 'Linux'
- name: Prepare executables for uploading (macOS)
run: |
mkdir executables
mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }}
mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }}
# Zip it so that signature is not lost
ditto -c -k --rsrc executables/subspace-farmer-${{ matrix.build.suffix }} executables/subspace-farmer-${{ matrix.build.suffix }}.zip
ditto -c -k --rsrc executables/subspace-node-${{ matrix.build.suffix }} executables/subspace-node-${{ matrix.build.suffix }}.zip
rm executables/subspace-farmer-${{ matrix.build.suffix }}
rm executables/subspace-node-${{ matrix.build.suffix }}
if: runner.os == 'macOS'
- name: Prepare executables for uploading (Windows)
run: |
mkdir executables
move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe executables/subspace-farmer-${{ matrix.build.suffix }}.exe
move ${{ env.PRODUCTION_TARGET }}/subspace-node.exe executables/subspace-node-${{ matrix.build.suffix }}.exe
if: runner.os == 'Windows'
- name: Upload node and farmer executables to artifacts
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.1.3
with:
name: executables-${{ matrix.build.suffix }}
path: |
executables/*
if-no-files-found: error
- name: Upload node and farmer executables to assets
uses: alexellis/upload-assets@13926a61cdb2cb35f5fdef1c06b8b591523236d3 # 0.4.1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
asset_paths: '["executables/*"]'
if: github.event_name == 'push' && github.ref_type == 'tag'