Merging main latest

Signed-off-by: AMZN-Gene <[email protected]>

.github/workflows/build-container.yaml

@@ -0,0 +1,73 @@
+# This automation builds a aarch64 Ubuntu container
+name: Build Container
+
+on:
+  # Allows you to run this workflow manually from the Actions tag
+  workflow_dispatch: 
+
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - .github/workflows/build-container.yaml
+
+
+jobs:
+  build-container:
+    name: Build Ubuntu container for aarch64
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      # We use ghcr.io to store the docker image cache for ARM builds
+      packages: write
+    steps:
+      - name: Run build command (aarch64)
+        uses: uraimo/[email protected]
+        with:
+          env: |
+            GITHUB_WORKFLOW: ${{ github.workflow }} # Sets the docker image to the name of the workflow
+          arch: aarch64
+          distro: ubuntu20.04
+          githubToken: ${{ github.token }}
+          shell: /bin/bash
+          install: |
+            ## Set variables. "env" not supported in install phase
+            export CLANG_VER=12
+            export GCC_VER=9
+            export SCCACHE_VER=0.5.4
+            export CMAKE_VER=3.27.6
+            ## Install build dependancies from apt
+            apt-get update
+            apt-get install -y build-essential curl git libssl-dev libffi-dev libbz2-dev libgdbm-compat-dev libgdbm-dev liblzma-dev libreadline-dev libtool \
+              ninja-build python3 python3-pip tcl8.6-dev tk8.6-dev texinfo software-properties-common wget
+            python3 -m pip install boto3 certifi
+            # Install Clang/GCC at specific version
+            apt-get install -y clang-${CLANG_VER} gcc-${GCC_VER} g++-${GCC_VER}
+            update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${CLANG_VER} 10
+            update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${CLANG_VER} 10
+            update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VER} 10
+            update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VER} 10
+            ## Install sccache
+            wget -qO- "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VER}/sccache-v${SCCACHE_VER}-aarch64-unknown-linux-musl.tar.gz" \
+              | tar xzf - -O --wildcards '*/sccache' > '/usr/local/bin/sccache' \
+              && chmod +x '/usr/local/bin/sccache'
+            ## Install cmake  
+            wget -q "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-linux-aarch64.sh" \
+              -O /tmp/cmake-install.sh \
+              && chmod u+x /tmp/cmake-install.sh \
+              && mkdir /opt/cmake-${CMAKE_VER} \
+              && /tmp/cmake-install.sh --skip-license --prefix=/opt/cmake-${CMAKE_VER} \
+              && rm /tmp/cmake-install.sh \
+              && ln -s /opt/cmake-${CMAKE_VER}/bin/* /usr/local/bin
+            rm -rf /var/lib/apt/lists/*
+          run: |
+            lsb_release -a
+            uname -a
+            gcc --version
+            g++ --version
+            clang --version
+            sccache --version
+            cmake --version
+            git --version
+            python3 --version

.github/workflows/build-package.yaml

@@ -0,0 +1,246 @@
+# This automation builds 3p packages based on a PR
+
+name: Build 3P Packages
+
+on:
+  pull_request:
+    branches:
+      - main
+      - development
+    paths:
+      - 'package_build_list_host_*.json'
+
+jobs:
+  detect-changes:
+    name: Detecting changes in PR to build
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.detect-platform.outputs.matrix }}
+
+    steps:
+    - name: Checkout 3P source repo
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Get package and platform from JSON changes
+      id: detect-platform
+      run: |
+        CHANGED_FILES=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} --name-only)
+        # Construct the package and os into a json string to be consumed by Github Actions runners
+        JSON="{\"include\":["
+        for FILE in $CHANGED_FILES; do
+          if [[ $FILE == package_build_list_host_* ]]; then
+            PLATFORM=$(echo $FILE | sed -n 's/package_build_list_host_\(.*\).json/\1/p')
+            case $PLATFORM in
+            linux*)
+              OS_RUNNER="ubuntu-20.04"
+              ;;
+            windows)
+              OS_RUNNER="windows-latest" # This is bundled with VS2022
+              ;;
+            darwin)
+              OS_RUNNER="macos-latest"
+              ;;
+            *)
+              OS_RUNNER="windows-latest" # default
+              ;;
+            esac
+
+            DIFF=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} --no-ext-diff --unified=0 \
+                        --exit-code -a --no-prefix -- $FILE | egrep "^\+" | grep Scripts) # Get oly the changes that can be built
+          
+            PACKAGE=$(echo $DIFF | cut -d'"' -f2)
+            PACKPATH=$(echo $DIFF | egrep -o "package-system/[^ ]*")
+            DOCKER=$(test -f "$PACKPATH/Dockerfile" && echo 1 || echo 0)
+            JSONline="{\"package\": \"$PACKAGE\", \"os\": \"$OS_RUNNER\", \"dockerfile\": \"$DOCKER\"},"
+            if [[ "$JSON" != *"$JSONline"* ]]; then
+              JSON="$JSON$JSONline"
+            fi
+          fi
+        done
+
+        # Remove last "," and add closing brackets
+        if [[ $JSON == *, ]]; then
+          JSON="${JSON%?}"
+        fi
+        JSON="$JSON]}"
+        echo $JSON
+
+        # Set output
+        echo "matrix=$( echo "$JSON" )" >> $GITHUB_OUTPUT
+
+  validate-changes:
+    name: Check changes for issues
+    needs: detect-changes
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout 3P source repo
+      uses: actions/checkout@v4
+
+    - name: Check if package already exists in prod
+      env:
+        PROD_CDN: ${{ vars.PROD_CDN }} # Change this to compare on your own endpoint
+      run: |
+        url="${{ env.PROD_CDN }}/${{ matrix.package }}"
+        if curl --head --silent --fail ${url}.tar.xz > /dev/null 2>&1; then
+          echo ${{ matrix.package }} already exists in prod. Check the rev in the json file to ensure it is incremented
+          exit 1
+        else
+          echo ${{ matrix.package }} does not exist in CDN, continuing...
+          exit 0
+        fi
+
+    - name: Malware scan of repo
+      uses: dell/common-github-actions/malware-scanner@main
+      with:
+        directories: .
+        options: -r
+
+  build-on-specific-os:
+    name: Build on "${{ matrix.os }}" for "${{ matrix.package }}"
+    needs: [detect-changes, validate-changes]
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - name: Configure
+      id: configure
+      run: |
+        git config --global core.longpaths true
+        echo "uid_gid=$(id -u):$(id -g)" >> $GITHUB_OUTPUT
+      
+    - name: Checkout 3P source repo
+      uses: actions/checkout@v4
+      with:
+        path: source
+        fetch-depth: 0
+
+    - name: Checkout 3P scripts repo
+      uses: actions/checkout@v4
+      with:
+        repository: o3de/3p-package-scripts
+        path: scripts
+
+    - name: Update python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+        cache: 'pip'
+
+    - name: Install python dependancies
+      run: |
+        python3 -m pip install boto3 certifi
+    
+    - name: Update cmake/ninja
+      uses: lukka/get-cmake@latest 
+
+    - name: Update msbuild path
+      if: runner.os == 'Windows'
+      uses: ilammy/[email protected]
+
+    - name: Install clang/gcc
+      if: runner.os == 'Linux'
+      env:
+        CLANG_VER: 12
+        GCC_VER: 9
+      run: |
+        sudo apt-get install -y clang-${{ env.CLANG_VER }} gcc-${{ env.GCC_VER }} g++-${{ env.GCC_VER }}
+        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ env.CLANG_VER }} 10
+        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${{ env.CLANG_VER }} 10
+        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ env.GCC_VER }} 10
+        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${{ env.GCC_VER }} 10
+
+    - name: Use sccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        variant: sccache
+        max-size: 2048M
+        key: ${{ matrix.package }}-${{ matrix.os }}
+        restore-keys:
+          ${{ matrix.package }}-${{ matrix.os }}  
+
+    - name: Set up QEMU (aarch64) # Only if the package folder contains a Dockerfile
+      if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile == '1') }}
+      run: |
+        sudo apt-get install -y qemu qemu-user-static
+
+    - name: Run build command
+      if: ${{ (!contains(matrix.package, 'aarch64')) || (matrix.dockerfile == '1') }}
+      env:
+        CMAKE_CXX_COMPILER_LAUNCHER: sccache
+        CMAKE_C_COMPILER_LAUNCHER: sccache
+        CMAKE_GENERATOR: Ninja # ccache/sccache cannot be used as the compiler launcher under cmake if the generator is MSBuild
+      run: |
+        python3 scripts/o3de_package_scripts/build_package.py --search_path source ${{ matrix.package }}
+
+    - name: Run build command (aarch64) # Generic build for packages without a Dockerfile
+      if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile != '1') }}
+      uses: uraimo/[email protected]
+      with:
+        arch: none
+        distro: none
+        base_image: ghcr.io/${{ github.repository }}/run-on-arch-${{ github.repository_owner }}-${{ github.event.repository.name }}-build-container-aarch64-ubuntu20-04:latest # built from build-container.yaml
+        setup: |
+          grep -q ${{ matrix.package }} ${PWD}/source/package_build_list_host_linux.json || rm ${PWD}/source/package_build_list_host_linux.json
+        dockerRunArgs: |
+          --platform=linux/arm64 
+          --user ${{ steps.configure.outputs.uid_gid }}
+          --volume "${PWD}:/workspace"
+          --volume "${PWD}/scripts:/scripts"
+          --volume "${PWD}/source:/source"
+        env: |
+          CMAKE_CXX_COMPILER_LAUNCHER: sccache
+          CMAKE_C_COMPILER_LAUNCHER: sccache
+          SCCACHE_IDLE_TIMEOUT: 0 
+          SCCACHE_DIR: /workspace/.sccache
+          SCCACHE_CACHE_SIZE: 2048M
+        shell: /bin/bash
+        run: |
+          lsb_release -a
+          uname -a
+          sccache --start-server
+          sccache -z
+          ls -lah /workspace
+          python3 /scripts/o3de_package_scripts/build_package.py --search_path /source/ ${{ matrix.package }}
+          
+    - name: Upload packages
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.package }}
+        path: source/packages/*
+
+  validate-packages:
+    name: Validating ${{ matrix.package }}
+    needs: [detect-changes, build-on-specific-os]
+    runs-on: 'ubuntu-latest'
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
+    steps:
+      - name: Download packages
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.package }}
+
+      - name: Verify SHA256
+        run: |
+          echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS)"
+          echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS | cut -d" " -f1) ${{ matrix.package }}.tar.xz" | sha256sum --check
+
+      - name: Decompress package
+        if: ${{ !contains(matrix.package, 'aarch64') }}
+        run: |
+          tar -xvf ${{ matrix.package }}.tar.xz
+    
+      - name: Malware scan
+        uses: dell/common-github-actions/malware-scanner@main
+        with:
+          directories: .
+          options: -r