Skip to content

Commit

Permalink
Merging main latest
Browse files Browse the repository at this point in the history
Signed-off-by: AMZN-Gene <[email protected]>
  • Loading branch information
AMZN-Gene committed Nov 30, 2023
2 parents cc2ad40 + a8019a5 commit 757df11
Show file tree
Hide file tree
Showing 22 changed files with 754 additions and 53 deletions.
73 changes: 73 additions & 0 deletions .github/workflows/build-container.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# This automation builds a aarch64 Ubuntu container
name: Build Container

on:
# Allows you to run this workflow manually from the Actions tag
workflow_dispatch:

push:
branches:
- main
- development
paths:
- .github/workflows/build-container.yaml


jobs:
build-container:
name: Build Ubuntu container for aarch64
runs-on: ubuntu-latest
permissions:
contents: read
# We use ghcr.io to store the docker image cache for ARM builds
packages: write
steps:
- name: Run build command (aarch64)
uses: uraimo/[email protected]
with:
env: |
GITHUB_WORKFLOW: ${{ github.workflow }} # Sets the docker image to the name of the workflow
arch: aarch64
distro: ubuntu20.04
githubToken: ${{ github.token }}
shell: /bin/bash
install: |
## Set variables. "env" not supported in install phase
export CLANG_VER=12
export GCC_VER=9
export SCCACHE_VER=0.5.4
export CMAKE_VER=3.27.6
## Install build dependancies from apt
apt-get update
apt-get install -y build-essential curl git libssl-dev libffi-dev libbz2-dev libgdbm-compat-dev libgdbm-dev liblzma-dev libreadline-dev libtool \
ninja-build python3 python3-pip tcl8.6-dev tk8.6-dev texinfo software-properties-common wget
python3 -m pip install boto3 certifi
# Install Clang/GCC at specific version
apt-get install -y clang-${CLANG_VER} gcc-${GCC_VER} g++-${GCC_VER}
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${CLANG_VER} 10
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${CLANG_VER} 10
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VER} 10
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VER} 10
## Install sccache
wget -qO- "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VER}/sccache-v${SCCACHE_VER}-aarch64-unknown-linux-musl.tar.gz" \
| tar xzf - -O --wildcards '*/sccache' > '/usr/local/bin/sccache' \
&& chmod +x '/usr/local/bin/sccache'
## Install cmake
wget -q "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-linux-aarch64.sh" \
-O /tmp/cmake-install.sh \
&& chmod u+x /tmp/cmake-install.sh \
&& mkdir /opt/cmake-${CMAKE_VER} \
&& /tmp/cmake-install.sh --skip-license --prefix=/opt/cmake-${CMAKE_VER} \
&& rm /tmp/cmake-install.sh \
&& ln -s /opt/cmake-${CMAKE_VER}/bin/* /usr/local/bin
rm -rf /var/lib/apt/lists/*
run: |
lsb_release -a
uname -a
gcc --version
g++ --version
clang --version
sccache --version
cmake --version
git --version
python3 --version
246 changes: 246 additions & 0 deletions .github/workflows/build-package.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@
# This automation builds 3p packages based on a PR

name: Build 3P Packages

on:
pull_request:
branches:
- main
- development
paths:
- 'package_build_list_host_*.json'

jobs:
detect-changes:
name: Detecting changes in PR to build
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.detect-platform.outputs.matrix }}

steps:
- name: Checkout 3P source repo
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Get package and platform from JSON changes
id: detect-platform
run: |
CHANGED_FILES=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} --name-only)
# Construct the package and os into a json string to be consumed by Github Actions runners
JSON="{\"include\":["
for FILE in $CHANGED_FILES; do
if [[ $FILE == package_build_list_host_* ]]; then
PLATFORM=$(echo $FILE | sed -n 's/package_build_list_host_\(.*\).json/\1/p')
case $PLATFORM in
linux*)
OS_RUNNER="ubuntu-20.04"
;;
windows)
OS_RUNNER="windows-latest" # This is bundled with VS2022
;;
darwin)
OS_RUNNER="macos-latest"
;;
*)
OS_RUNNER="windows-latest" # default
;;
esac
DIFF=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} --no-ext-diff --unified=0 \
--exit-code -a --no-prefix -- $FILE | egrep "^\+" | grep Scripts) # Get oly the changes that can be built
PACKAGE=$(echo $DIFF | cut -d'"' -f2)
PACKPATH=$(echo $DIFF | egrep -o "package-system/[^ ]*")
DOCKER=$(test -f "$PACKPATH/Dockerfile" && echo 1 || echo 0)
JSONline="{\"package\": \"$PACKAGE\", \"os\": \"$OS_RUNNER\", \"dockerfile\": \"$DOCKER\"},"
if [[ "$JSON" != *"$JSONline"* ]]; then
JSON="$JSON$JSONline"
fi
fi
done
# Remove last "," and add closing brackets
if [[ $JSON == *, ]]; then
JSON="${JSON%?}"
fi
JSON="$JSON]}"
echo $JSON
# Set output
echo "matrix=$( echo "$JSON" )" >> $GITHUB_OUTPUT
validate-changes:
name: Check changes for issues
needs: detect-changes
strategy:
fail-fast: false
matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
runs-on: ubuntu-latest

steps:
- name: Checkout 3P source repo
uses: actions/checkout@v4

- name: Check if package already exists in prod
env:
PROD_CDN: ${{ vars.PROD_CDN }} # Change this to compare on your own endpoint
run: |
url="${{ env.PROD_CDN }}/${{ matrix.package }}"
if curl --head --silent --fail ${url}.tar.xz > /dev/null 2>&1; then
echo ${{ matrix.package }} already exists in prod. Check the rev in the json file to ensure it is incremented
exit 1
else
echo ${{ matrix.package }} does not exist in CDN, continuing...
exit 0
fi
- name: Malware scan of repo
uses: dell/common-github-actions/malware-scanner@main
with:
directories: .
options: -r

build-on-specific-os:
name: Build on "${{ matrix.os }}" for "${{ matrix.package }}"
needs: [detect-changes, validate-changes]
strategy:
fail-fast: false
matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
runs-on: ${{ matrix.os }}

steps:
- name: Configure
id: configure
run: |
git config --global core.longpaths true
echo "uid_gid=$(id -u):$(id -g)" >> $GITHUB_OUTPUT
- name: Checkout 3P source repo
uses: actions/checkout@v4
with:
path: source
fetch-depth: 0

- name: Checkout 3P scripts repo
uses: actions/checkout@v4
with:
repository: o3de/3p-package-scripts
path: scripts

- name: Update python
uses: actions/setup-python@v4
with:
python-version: '3.10'
cache: 'pip'

- name: Install python dependancies
run: |
python3 -m pip install boto3 certifi
- name: Update cmake/ninja
uses: lukka/get-cmake@latest

- name: Update msbuild path
if: runner.os == 'Windows'
uses: ilammy/[email protected]

- name: Install clang/gcc
if: runner.os == 'Linux'
env:
CLANG_VER: 12
GCC_VER: 9
run: |
sudo apt-get install -y clang-${{ env.CLANG_VER }} gcc-${{ env.GCC_VER }} g++-${{ env.GCC_VER }}
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ env.CLANG_VER }} 10
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${{ env.CLANG_VER }} 10
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ env.GCC_VER }} 10
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${{ env.GCC_VER }} 10
- name: Use sccache
uses: hendrikmuhs/[email protected]
with:
variant: sccache
max-size: 2048M
key: ${{ matrix.package }}-${{ matrix.os }}
restore-keys:
${{ matrix.package }}-${{ matrix.os }}

- name: Set up QEMU (aarch64) # Only if the package folder contains a Dockerfile
if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile == '1') }}
run: |
sudo apt-get install -y qemu qemu-user-static
- name: Run build command
if: ${{ (!contains(matrix.package, 'aarch64')) || (matrix.dockerfile == '1') }}
env:
CMAKE_CXX_COMPILER_LAUNCHER: sccache
CMAKE_C_COMPILER_LAUNCHER: sccache
CMAKE_GENERATOR: Ninja # ccache/sccache cannot be used as the compiler launcher under cmake if the generator is MSBuild
run: |
python3 scripts/o3de_package_scripts/build_package.py --search_path source ${{ matrix.package }}
- name: Run build command (aarch64) # Generic build for packages without a Dockerfile
if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile != '1') }}
uses: uraimo/[email protected]
with:
arch: none
distro: none
base_image: ghcr.io/${{ github.repository }}/run-on-arch-${{ github.repository_owner }}-${{ github.event.repository.name }}-build-container-aarch64-ubuntu20-04:latest # built from build-container.yaml
setup: |
grep -q ${{ matrix.package }} ${PWD}/source/package_build_list_host_linux.json || rm ${PWD}/source/package_build_list_host_linux.json
dockerRunArgs: |
--platform=linux/arm64
--user ${{ steps.configure.outputs.uid_gid }}
--volume "${PWD}:/workspace"
--volume "${PWD}/scripts:/scripts"
--volume "${PWD}/source:/source"
env: |
CMAKE_CXX_COMPILER_LAUNCHER: sccache
CMAKE_C_COMPILER_LAUNCHER: sccache
SCCACHE_IDLE_TIMEOUT: 0
SCCACHE_DIR: /workspace/.sccache
SCCACHE_CACHE_SIZE: 2048M
shell: /bin/bash
run: |
lsb_release -a
uname -a
sccache --start-server
sccache -z
ls -lah /workspace
python3 /scripts/o3de_package_scripts/build_package.py --search_path /source/ ${{ matrix.package }}
- name: Upload packages
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.package }}
path: source/packages/*

validate-packages:
name: Validating ${{ matrix.package }}
needs: [detect-changes, build-on-specific-os]
runs-on: 'ubuntu-latest'
strategy:
fail-fast: false
matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
steps:
- name: Download packages
uses: actions/download-artifact@v3
with:
name: ${{ matrix.package }}

- name: Verify SHA256
run: |
echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS)"
echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS | cut -d" " -f1) ${{ matrix.package }}.tar.xz" | sha256sum --check
- name: Decompress package
if: ${{ !contains(matrix.package, 'aarch64') }}
run: |
tar -xvf ${{ matrix.package }}.tar.xz
- name: Malware scan
uses: dell/common-github-actions/malware-scanner@main
with:
directories: .
options: -r
Loading

0 comments on commit 757df11

Please sign in to comment.