From 59ea110168a7af7614d589b9b668bf84879267dd Mon Sep 17 00:00:00 2001 From: Yaron Date: Thu, 7 Sep 2023 21:42:16 +0700 Subject: [PATCH] Add workflow for GitHub action from master. --- .github/workflows/ci-image-dev.yml | 44 +++++++++++++++++ .github/workflows/ci-image-release.yml | 66 ++++++++++++++++++++++++++ .github/workflows/ci-image-test.yml | 63 ++++++++++++++++++++++++ Makefile | 5 +- docker/Dockerfile | 21 +++++--- docker/Dockerfile.test | 17 +++++++ 6 files changed, 208 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/ci-image-dev.yml create mode 100644 .github/workflows/ci-image-release.yml create mode 100644 .github/workflows/ci-image-test.yml create mode 100644 docker/Dockerfile.test diff --git a/.github/workflows/ci-image-dev.yml b/.github/workflows/ci-image-dev.yml new file mode 100644 index 000000000..fd73ac2c7 --- /dev/null +++ b/.github/workflows/ci-image-dev.yml @@ -0,0 +1,44 @@ +name: CI - Development + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + push: + branches-ignore: [] + +jobs: + push-to-ecr: + permissions: + id-token: write + contents: write + name: build + runs-on: docker + steps: + - name: Clean environment + # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). + shell: bash + run: | + docker system df + docker system prune -a -f --filter "until=168h" + docker system df + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Docker build and push + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla + tag-length: 8 + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + cache-key: ${{ github.event.repository.name }} diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml new file mode 100644 index 000000000..f112ff90d --- /dev/null +++ b/.github/workflows/ci-image-release.yml @@ -0,0 +1,66 @@ +name: CI - Release + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + push: + tags: + - v* + +jobs: + release-image: + permissions: + id-token: write + contents: write + runs-on: docker + steps: + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Check if the commit or tag was passed manually + id: set-tag + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "tag=$(git rev-parse HEAD | cut -c1-8)" >> $GITHUB_OUTPUT + else + echo "tag=${{ github.ref_name }}" >> $GITHUB_OUTPUT + fi + shell: bash + - name: Check if the tag head is the same as release-v0.13.4 head + id: check-latest + if: github.event_name != 'workflow_dispatch' + run: | + if test $(git rev-parse origin/release-v0.13.4) = $(git rev-parse HEAD); then + echo "latest=true" >> $GITHUB_OUTPUT + else + echo "latest=false" >> $GITHUB_OUTPUT + fi + shell: bash + - name: Docker build and push + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} + tag-latest: ${{ steps.check-latest.outputs.latest }} + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + cache-key: ${{ github.event.repository.name }} + - name: Docker build and push (Dockerhub) + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }} + tag-latest: ${{ steps.check-latest.outputs.latest }} + cache-key: ${{ github.event.repository.name }} + registry-username: ${{ secrets.DOCKERHUB_USERNAME }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml new file mode 100644 index 000000000..6150e4836 --- /dev/null +++ b/.github/workflows/ci-image-test.yml @@ -0,0 +1,63 @@ +name: CI - Integration tests + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + pull_request: + branches: + - 'release-v**' + +jobs: + run-tests: + permissions: + id-token: write + contents: write + name: tests + runs-on: docker + steps: + - name: Clean environment + # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). + shell: bash + run: | + docker system df + docker system prune -a -f --filter "until=336h" + docker system df + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Configure AWS Credentials + uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + - name: Login to the registry + uses: docker/login-action@v2 + with: + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + - name: Build Docker images + run: | + DOCKER_BUILDKIT=1 docker build --target test_runner -t scilla:tests -f docker/Dockerfile . + shell: bash + - name: Run make test + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' + shell: bash + - name: Run make test_server + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' + shell: bash + - name: Run make coveralls + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}' + shell: bash + - name: Run make lint + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' + shell: bash diff --git a/Makefile b/Makefile index 95de4d851..4cac0ff7f 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,10 @@ clean: dune clean # Remove remaining files/folders ignored by git as defined in .gitignore (-X) # but keeping a local opam switch and other dependencies built. - git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed + @if [ -d .git ]; \ + then \ + git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed; \ + fi # Clean up libff installation cleanall: clean diff --git a/docker/Dockerfile b/docker/Dockerfile index 1e88c5e13..fc6bd691a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -47,7 +47,6 @@ FROM base AS builder ARG CMAKE_VERSION=3.25.1 RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \ - && echo "6598da34f0e3a0f763809e25cfdd646aa1d5e4d133c4277821e63ae5cfe09457 cmake-${CMAKE_VERSION}-Linux-x86_64.sh" | sha256sum -c \ && mkdir -p "${HOME}"/.local \ && bash ./cmake-${CMAKE_VERSION}-Linux-x86_64.sh --skip-license --prefix="${HOME}"/.local/ \ && "${HOME}"/.local/bin/cmake --version \ @@ -100,10 +99,9 @@ RUN apt update -y \ ENV VCPKG_BINARY_SOURCES="default,readwrite;x-gcs,gs://vcpkg/ubuntu/22.04/x64-linux-dynamic/${VCPKG_COMMIT_OR_TAG}/,read" +# Make sure vcpkg installs brings in the dependencies RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 - ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/lib/pkgconfig" RUN make opamdep-ci \ @@ -115,11 +113,20 @@ ARG BUILD_DIR="${SOURCE_DIR}/_build/default" ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/x64-linux-dynamic/lib" RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \ - && ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} \ - && rm -rf vcpkg_installed \ + && ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} + +FROM builder AS test_runner + +ENV VCPKG_ROOT=/vcpkg +RUN apt update -y && apt install -y sudo +RUN ./scripts/install_shellcheck_ubuntu.sh + +FROM builder AS cleanup_vcpkg + +RUN rm -rf vcpkg_installed \ && ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed -FROM ubuntu:22.04 +FROM ubuntu:22.04 AS base RUN apt-get update -y \ && apt-get install -y build-essential \ @@ -128,5 +135,5 @@ RUN apt-get update -y \ ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}" -COPY --from=builder ${SOURCE_DIR} ${SOURCE_DIR} +COPY --from=cleanup_vcpkg ${SOURCE_DIR} ${SOURCE_DIR} diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test new file mode 100644 index 000000000..f1e533cb7 --- /dev/null +++ b/docker/Dockerfile.test @@ -0,0 +1,17 @@ +ARG ACCOUNT_ID + +FROM ${ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 + +ENV VCPKG_ROOT="/vcpkg" +ENV SCILLA_REPO_ROOT="/scilla/0" + +WORKDIR /scilla/0/ +COPY . /scilla/0/ + +RUN apt update \ + && apt install -y sudo + +RUN eval $(opam env) \ + && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes + +RUN ./scripts/install_shellcheck_ubuntu.sh