Add CI for building and testing UMD (#70)

* first version of cis, probably don't work

* minor fix

* docker image build

* Fix build image

* fix build image v2

* remove push

* revert push, workflow_dispatch works on main only

* fix build-tests, remove push for build-image

* fix docker image used, remove gitlab-ci, remove grayskull until it starts working

* tar directly what is needed without install

* modify jobs a bit

* try show all files

* working forward...

* fix artifacts. use self hosted machine with a specific card.

* minor fix

* add forgotten needs. split build tests and run tests

* maybe fix untar

* cleanup tt-umd from last run

* some problem with dirs

* tar non-absolute paths

* minor fix

* fix untar v14563

* upload lib as well

* add deps to tar

* some path printout to revert for testing

* try path fix

* try another path fix, and see cluster descriptor

* upload cluster descriptor

* Remove comments from clusterdescr, fix passing tens device to image, add separate optional workflow

* add custom timeouts

* reduce source and tests into build-target

* addressed comments. Reduced dockerfile, reduced docker build image tags, fixed upload-artifacts argument, reduced common code in generate_cluster_desc, added comments for commented out archs

* minor fix

* yes, another minor fixes

* update on-push.yml as well, remove push trigger for building docker image

.github/docker_install_common.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Install build and runtime deps
+apt-get update && apt-get install -y \
+    software-properties-common \
+    build-essential \
+    cmake \
+    ninja-build \
+    git \
+    libhwloc-dev \
+    libgtest-dev \
+    libyaml-cpp-dev \
+    libboost-all-dev \
+    wget
+
+# Install clang 17
+wget https://apt.llvm.org/llvm.sh && \
+    chmod u+x llvm.sh && \
+    ./llvm.sh 17 && \
+    apt install -y libc++-17-dev libc++abi-17-dev && \
+    ln -s /usr/bin/clang-17 /usr/bin/clang && \
+    ln -s /usr/bin/clang++-17 /usr/bin/clang++
+
+# Install clang-format
+apt install -y clang-format-17 && \
+    ln -s /usr/bin/clang-format-17 /usr/bin/clang-format

.github/ubuntu-20.04.Dockerfile

@@ -0,0 +1,6 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY docker_install_common.sh /docker_install_common.sh
+RUN chmod +x /docker_install_common.sh && /docker_install_common.sh

.github/ubuntu-22.04.Dockerfile

@@ -0,0 +1,7 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY docker_install_common.sh /docker_install_common.sh
+RUN chmod +x /docker_install_common.sh && /docker_install_common.sh
+

.github/workflows/build-image.yml

@@ -0,0 +1,52 @@
+# This workflow is intended to be called manually when a new Docker image is needed.
+name: Build and Publish Docker Image
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  build:
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [
+          {name: ubuntu-22.04},
+          {name: ubuntu-20.04},
+        ]
+
+    name: Building docker image ${{ matrix.build.name }}
+    runs-on: ${{ matrix.build.name }}
+
+    steps:
+      - name: Set environment variable
+        run: echo "CI_IMAGE_NAME=ghcr.io/${{ github.repository }}/tt-umd-ci-${{ matrix.build.name }}" >> $GITHUB_ENV
+
+      - name: Fix permissions
+        run: sudo chmod 777 -R $GITHUB_WORKSPACE
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and export base Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .github
+          file: .github/${{ matrix.build.name }}.Dockerfile
+          push: true
+          build-args: |
+            GIT_SHA=${{ github.sha }}
+          tags: |
+            ${{ env.CI_IMAGE_NAME}}:${{ github.sha }}
+            ${{ env.CI_IMAGE_NAME}}:latest

.github/workflows/build-target.yml

@@ -0,0 +1,100 @@
+# Builds requested target and uploads its artifact if requested.
+# Build is performed on the specified architecture and on all supported OS versions.
+name: Build Target
+
+on:
+  workflow_call:
+    inputs:
+      arch:
+        required: true
+        type: string
+      timeout:
+        required: true
+        type: number
+      build-target:
+        required: true
+        type: string
+      upload-artifacts:
+        required: false
+        type: boolean
+        default: false
+  workflow_dispatch:
+    inputs:
+      arch:
+        required: true
+        description: 'The architecture to build for'
+        type: choice
+        options:
+          - grayskull
+          - wormhole_b0
+          - blackhole
+      timeout:
+        required: true
+        description: 'The timeout for the build job in minutes'
+        type: number
+      build-target:
+        required: true
+        description: 'The target to build'
+        type: choice
+        options:
+          - umd_device
+          - umd_tests
+      upload-artifacts:
+        required: false
+        description: 'Whether to upload artifacts'
+        type: boolean
+        default: false
+
+
+env:
+  BUILD_OUTPUT_DIR: ./build
+  LIB_OUTPUT_DIR: ./build/lib
+  DEPS_OUTPUT_DIR: ./build/_deps
+  TEST_OUTPUT_DIR: ./build/test
+  CREATE_MAP_BINARIES_DIR: ./device/bin/silicon
+
+jobs:
+  build:
+    timeout-minutes: ${{ inputs.timeout }}
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [
+          {runs-on: ubuntu-22.04, docker-image: tt-umd-ci-ubuntu-22.04},
+          {runs-on: ubuntu-20.04, docker-image: tt-umd-ci-ubuntu-20.04},
+        ]
+
+    name: Build ${{ inputs.build-target }} for ${{ inputs.arch }} on ${{ matrix.build.runs-on }}
+    runs-on: ${{ matrix.build.runs-on }}
+    container:
+      image: ghcr.io/${{ github.repository }}/${{ matrix.build.docker-image }}:latest
+      options: --user root
+
+    env:
+      ARCH_NAME: ${{ inputs.arch }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Build ${{ inputs.build-target }}
+      run: |
+        echo "Compiling the code..."
+        cmake -B ${{ env.BUILD_OUTPUT_DIR }} -G Ninja
+        cmake --build ${{ env.BUILD_OUTPUT_DIR }} --target ${{ inputs.build-target }}
+        echo "Compile complete."
+
+    # This is needed to preserve file permissions
+    # https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
+    - name: Tar build, test and run artifacts
+      if: ${{ inputs.upload-artifacts }}
+      shell: bash
+      run: tar cvf artifact.tar ${{ env.TEST_OUTPUT_DIR }} ${{ env.LIB_OUTPUT_DIR }} ${{ env.DEPS_OUTPUT_DIR }} ${{ env.CREATE_MAP_BINARIES_DIR }}
+
+    - name: Upload build artifacts archive
+      if: ${{ inputs.upload-artifacts }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: build-artifacts-${{ inputs.arch }}-${{ matrix.build.runs-on }}
+        path: artifact.tar

.github/workflows/on-pr-opt.yml

@@ -0,0 +1,47 @@
+# Optional PR checks
+name: On PR - Optional
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build-tests:
+    secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          # Enable once we have functional cards with specified architecture.
+          # { arch: grayskull },
+          { arch: wormhole_b0 },
+          # { arch: blackhole },
+        ]
+    uses: ./.github/workflows/build-target.yml
+    with:
+      arch: ${{ matrix.test-group.arch }}
+      timeout: 10
+      build-target: umd_tests
+      upload-artifacts: true
+
+  test-all:
+    secrets: inherit
+    needs: build-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          # Enable once we have functional cards.
+          # { arch: grayskull, card: e75 },
+          # { arch: grayskull, card: e150 },
+          # { arch: grayskull, card: e300 },
+          { arch: wormhole_b0, card: n150, timeout: 5 },
+          { arch: wormhole_b0, card: n300, timeout: 15 },
+          # { arch: blackhole },
+        ]
+    uses: ./.github/workflows/run-tests.yml
+    with:
+      arch: ${{ matrix.test-group.arch }}
+      card: ${{ matrix.test-group.card }}
+      timeout: ${{ matrix.test-group.timeout }}

.github/workflows/on-pr.yml

@@ -0,0 +1,24 @@
+# Mandatory PR checks
+name: On PR
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build-all:
+    secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          { arch: grayskull },
+          { arch: wormhole_b0 },
+          { arch: blackhole },
+        ]
+    uses: ./.github/workflows/build-target.yml
+    with:
+      arch: ${{ matrix.test-group.arch }}
+      timeout: 10
+      build-target: umd_device

.github/workflows/on-push.yml

@@ -0,0 +1,62 @@
+name: On Push
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "main" ]
+
+jobs:
+  build-all:
+    secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          { arch: grayskull },
+          { arch: wormhole_b0 },
+          { arch: blackhole },
+        ]
+    uses: ./.github/workflows/build-target.yml
+    with:
+      arch: ${{ matrix.test-group.arch }}
+      timeout: 10
+      build-target: umd_device
+
+  build-tests:
+    secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          # Enable once we have functional cards with specified architecture.
+          # { arch: grayskull },
+          { arch: wormhole_b0 },
+          # { arch: blackhole },
+        ]
+    uses: ./.github/workflows/build-target.yml
+    with:
+      arch: ${{ matrix.test-group.arch }}
+      timeout: 10
+      build-target: umd_tests
+      upload-artifacts: true
+
+  test-all:
+    secrets: inherit
+    needs: build-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group: [
+          # Enable once we have functional cards.
+          # { arch: grayskull, card: e75 },
+          # { arch: grayskull, card: e150 },
+          # { arch: grayskull, card: e300 },
+          { arch: wormhole_b0, card: n150, timeout: 5 },
+          { arch: wormhole_b0, card: n300, timeout: 15 },
+          # { arch: blackhole },
+        ]
+    uses: ./.github/workflows/run-tests.yml
+    with:
+      arch: ${{ matrix.test-group.arch }}
+      card: ${{ matrix.test-group.card }}
+      timeout: ${{ matrix.test-group.timeout }}