From e48360acc334c8a3e9f8319374af015f731dba31 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Thu, 15 Jun 2023 17:21:36 +0400 Subject: [PATCH 01/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/ci-image-test.yml | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/ci-image-test.yml diff --git a/.github/ci-image-test.yml b/.github/ci-image-test.yml new file mode 100644 index 000000000..325ce567f --- /dev/null +++ b/.github/ci-image-test.yml @@ -0,0 +1,45 @@ +name: CI - Integration tests + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + pull_request: + branches: + - 'master' + +jobs: + run-tests: + name: tests + runs-on: docker + steps: + - name: Clean environment + # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). + shell: bash + run: | + docker system df + docker system prune -a -f --filter "until=336h" + docker system df + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Build Docker images + run: DOCKER_BUILDKIT=1 docker build --build-arg EXTRA_BUILD_PARAMS="debug tests" --target test_runner -t scilla:tests -f docker/Dockerfile . + shell: bash + - name: Run make test + run: | + docker run --rm -i scilla:tests -- cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test + shell: bash + - name: Run make test_server + run: | + docker run --rm -i scilla:tests -- cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server + shell: bash + - name: Run make lint + run: | + docker run --rm -i scilla:tests -- cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint + shell: bash From 0194559357d3649fc9f4784c41fa99366a958dbe Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Thu, 15 Jun 2023 17:28:21 +0400 Subject: [PATCH 02/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/{ => workflows}/ci-image-test.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/ci-image-test.yml (100%) diff --git a/.github/ci-image-test.yml b/.github/workflows/ci-image-test.yml similarity index 100% rename from .github/ci-image-test.yml rename to .github/workflows/ci-image-test.yml From 18377c9bdcf1d01ec525b1c87761b20ae443ad6e Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 09:35:34 +0400 Subject: [PATCH 03/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 325ce567f..91bd31e3c 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -29,7 +29,7 @@ jobs: fetch-depth: 0 ref: ${{ inputs.commitOrTag }} - name: Build Docker images - run: DOCKER_BUILDKIT=1 docker build --build-arg EXTRA_BUILD_PARAMS="debug tests" --target test_runner -t scilla:tests -f docker/Dockerfile . + run: DOCKER_BUILDKIT=1 docker build -t scilla:tests -f docker/Dockerfile . shell: bash - name: Run make test run: | From 231a4c433265fba8bd57a2202da248a090b029f5 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 09:44:12 +0400 Subject: [PATCH 04/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 91bd31e3c..f70b4fb79 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -33,13 +33,13 @@ jobs: shell: bash - name: Run make test run: | - docker run --rm -i scilla:tests -- cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test + docker run --rm -i scilla:tests -- cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test shell: bash - name: Run make test_server run: | - docker run --rm -i scilla:tests -- cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server + docker run --rm -i scilla:tests -- cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server shell: bash - name: Run make lint run: | - docker run --rm -i scilla:tests -- cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint + docker run --rm -i scilla:tests -- cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint shell: bash From 331eee255a34aeb50552bd93e4f89020807b89f7 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 11:34:58 +0400 Subject: [PATCH 05/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-dev.yml | 53 ++++++++++++++++++++++++ .github/workflows/ci-image-release.yml | 57 ++++++++++++++++++++++++++ .github/workflows/ci-image-test.yml | 10 +++-- docker/Dockerfile.test | 18 ++++++++ 4 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/ci-image-dev.yml create mode 100644 .github/workflows/ci-image-release.yml create mode 100644 docker/Dockerfile.test diff --git a/.github/workflows/ci-image-dev.yml b/.github/workflows/ci-image-dev.yml new file mode 100644 index 000000000..96bea9cb5 --- /dev/null +++ b/.github/workflows/ci-image-dev.yml @@ -0,0 +1,53 @@ +name: CI - Development + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + push: + branches-ignore: + - 'release/**' + +jobs: + push-to-ecr: + permissions: + id-token: write + contents: write + name: build + runs-on: docker + steps: + - name: Clean environment + # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). + shell: bash + run: | + docker system df + docker system prune -a -f --filter "until=168h" + docker system df + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Configure AWS Credentials + uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + - name: Login to the registry + uses: docker/login-action@v2 + with: + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + - name: Image tag + id: set-tag + run: echo "tag=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + shell: bash + - name: Build Docker images + run: DOCKER_BUILDKIT=1 docker build -t ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} -f docker/Dockerfile . + shell: bash + - name: Push Docker images + run: docker push ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} + shell: bash diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml new file mode 100644 index 000000000..78d5928ec --- /dev/null +++ b/.github/workflows/ci-image-release.yml @@ -0,0 +1,57 @@ +scillaname: CI - Release + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + push: + branches: + - 'release/**' + paths: + - 'VERSION' + tags: + - v* + +jobs: + release-image: + permissions: + id-token: write + contents: write + runs-on: docker + steps: + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Configure AWS Credentials + uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + - name: Login to the registry + uses: docker/login-action@v2 + with: + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + - name: Image tag + id: set-tag + run: echo "tag=v$(tail -n +2 VERSION | head -n1).$(tail -n +4 VERSION | head -n1).$(tail -n +6 VERSION | head -n1)" >> $GITHUB_OUTPUT + shell: bash + - name: Build Docker images + run: DOCKER_BUILDKIT=1 docker build -t ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.comscilla:${{ steps.set-tag.outputs.tag }} -t zilliqa/scilla:${{ steps.set-tag.outputs.tag }} -f docker/Dockerfile . + shell: bash + - name: Push Docker images + run: docker push ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.comscilla:${{ steps.set-tag.outputs.tag }} + shell: bash + - name: Login to the DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push Docker images + run: docker push zilliqa/scilla:${{ steps.set-tag.outputs.tag }} + shell: bash diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index f70b4fb79..f04c4e0f6 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -29,17 +29,19 @@ jobs: fetch-depth: 0 ref: ${{ inputs.commitOrTag }} - name: Build Docker images - run: DOCKER_BUILDKIT=1 docker build -t scilla:tests -f docker/Dockerfile . + run: | + DOCKER_BUILDKIT=1 docker build -t scilla:local -f docker/Dockerfile . + DOCKER_BUILDKIT=1 docker build -t scilla:tests -f docker/Dockerfile.test . shell: bash - name: Run make test run: | - docker run --rm -i scilla:tests -- cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test + docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' shell: bash - name: Run make test_server run: | - docker run --rm -i scilla:tests -- cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server + docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' shell: bash - name: Run make lint run: | - docker run --rm -i scilla:tests -- cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint + docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' shell: bash diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test new file mode 100644 index 000000000..a11c0090c --- /dev/null +++ b/docker/Dockerfile.test @@ -0,0 +1,18 @@ +# Requires a Scilla local tag build from docker/Dockerfile +FROM scilla:local + +RUN apt-get update -y \ + && apt install -y sudo + +COPY / /scilla/0/ + +RUN cd /scilla/0 \ + && eval \$(opam env) \ + && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes" + +ARG SC_VERSION=v0.7.1 + +RUN wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${SC_VERSION}/shellcheck-${SC_VERSION}.linux.x86_64.tar.xz" | tar -xJv \ + && sudo mv "shellcheck-${SC_VERSION}/shellcheck" /usr/local/bin/ \ + && rm -rf "shellcheck-${SC_VERSION}/shellcheck" \ + && shellcheck --version From e3c42a76433c45aad9cb52cf3a4e48738eb851db Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 11:35:55 +0400 Subject: [PATCH 06/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- docker/Dockerfile.test | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index a11c0090c..c059045d1 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -10,9 +10,4 @@ RUN cd /scilla/0 \ && eval \$(opam env) \ && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes" -ARG SC_VERSION=v0.7.1 - -RUN wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${SC_VERSION}/shellcheck-${SC_VERSION}.linux.x86_64.tar.xz" | tar -xJv \ - && sudo mv "shellcheck-${SC_VERSION}/shellcheck" /usr/local/bin/ \ - && rm -rf "shellcheck-${SC_VERSION}/shellcheck" \ - && shellcheck --version +RUN ./scripts/install_shellcheck_ubuntu.sh From ee40f6235d576173753f682a05a352a6dc50883f Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 12:03:55 +0400 Subject: [PATCH 07/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- docker/Dockerfile.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index c059045d1..6c3224454 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -7,7 +7,7 @@ RUN apt-get update -y \ COPY / /scilla/0/ RUN cd /scilla/0 \ - && eval \$(opam env) \ + && eval $(opam env) \ && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes" RUN ./scripts/install_shellcheck_ubuntu.sh From 53b7c41d3bb252bd9d8d1545a18362940d59dcdb Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 12:15:59 +0400 Subject: [PATCH 08/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- docker/Dockerfile.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 6c3224454..d9a9cff2a 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -8,6 +8,6 @@ COPY / /scilla/0/ RUN cd /scilla/0 \ && eval $(opam env) \ - && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes" + && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes RUN ./scripts/install_shellcheck_ubuntu.sh From 16c20eff71def354bd3c62cca69ecb8c90e03e06 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 12:32:13 +0400 Subject: [PATCH 09/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- Jenkinsfile.disabled | 76 ++++++++++++++++++++++++++++++++++++++++++ docker/Dockerfile.test | 3 ++ 2 files changed, 79 insertions(+) create mode 100644 Jenkinsfile.disabled diff --git a/Jenkinsfile.disabled b/Jenkinsfile.disabled new file mode 100644 index 000000000..ac7cfc28c --- /dev/null +++ b/Jenkinsfile.disabled @@ -0,0 +1,76 @@ +def podDefinition = """ +apiVersion: "v1" +kind: "Pod" +spec: + nodeSelector: + beta.kubernetes.io/os: "linux" + containers: + - name: "ubuntu" + image: "648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9" + command: + - cat + tty: true + resources: + requests: + ephemeral-storage: "20Gi" + limits: + ephemeral-storage: "20Gi" +""" + +String skipciMessage = 'Aborting because commit message contains [skip ci]' + +timestamps { + ansiColor('gnome-terminal') { + podTemplate(yaml: podDefinition) { + timeout(time: 60, unit: 'MINUTES') { + node(POD_LABEL) { + try { + stage('Checkout scm') { + checkout scm + def pr_skipci = "0" + try { + if (env.CHANGE_TITLE != null && env.CHANGE_TITLE != "") { + pr_skipci = sh(script: "echo ${env.CHANGE_TITLE.replace("(","").replace(")","")} | fgrep -ie '[skip ci]' -e '[ci skip]' | wc -l", returnStdout: true).trim() + } + } catch (err) { + println err.getMessage() + error("Error reading the Pull Request title, please check and eventually remove special characters") + } + def skipci = sh(script: "git log -1 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' | wc -l", returnStdout: true).trim() + if (skipci != "0" || pr_skipci != "0") { + error(skipciMessage) + } + } + container('ubuntu') { + env.VCPKG_ROOT="/vcpkg" + env.SCILLA_REPO_ROOT="/scilla/0" + stage('Update source code') { + sh "mkdir -p /scilla/0 && cp -r ${WORKSPACE}/* /scilla/0 && ln -s ${WORKSPACE}/.git /scilla/0/.git" + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes" + sh "apt update && apt install -y sudo" + sh "./scripts/install_shellcheck_ubuntu.sh" + } + stage('Test') { + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test" + } + stage('Test server') { + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server" + } + stage('Coveralls') { + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${BUILD_NUMBER}" + } + stage('Lint') { + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint" + } + } + } catch (err) { + if (err.getMessage() == skipciMessage) + currentBuild.result = 'SUCCESS' + else + throw err + } + } + } + } + } +} diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index d9a9cff2a..de62613ee 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,6 +1,9 @@ # Requires a Scilla local tag build from docker/Dockerfile FROM scilla:local +ENV VCPKG_ROOT="/vcpkg" +ENV SCILLA_REPO_ROOT="/scilla/0" + RUN apt-get update -y \ && apt install -y sudo From 268d1d92f43fc50d8ceaf9e3b311c3b541c53d8f Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 12:34:56 +0400 Subject: [PATCH 10/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- Jenkinsfile | 76 ----------------------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index ac7cfc28c..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,76 +0,0 @@ -def podDefinition = """ -apiVersion: "v1" -kind: "Pod" -spec: - nodeSelector: - beta.kubernetes.io/os: "linux" - containers: - - name: "ubuntu" - image: "648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9" - command: - - cat - tty: true - resources: - requests: - ephemeral-storage: "20Gi" - limits: - ephemeral-storage: "20Gi" -""" - -String skipciMessage = 'Aborting because commit message contains [skip ci]' - -timestamps { - ansiColor('gnome-terminal') { - podTemplate(yaml: podDefinition) { - timeout(time: 60, unit: 'MINUTES') { - node(POD_LABEL) { - try { - stage('Checkout scm') { - checkout scm - def pr_skipci = "0" - try { - if (env.CHANGE_TITLE != null && env.CHANGE_TITLE != "") { - pr_skipci = sh(script: "echo ${env.CHANGE_TITLE.replace("(","").replace(")","")} | fgrep -ie '[skip ci]' -e '[ci skip]' | wc -l", returnStdout: true).trim() - } - } catch (err) { - println err.getMessage() - error("Error reading the Pull Request title, please check and eventually remove special characters") - } - def skipci = sh(script: "git log -1 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' | wc -l", returnStdout: true).trim() - if (skipci != "0" || pr_skipci != "0") { - error(skipciMessage) - } - } - container('ubuntu') { - env.VCPKG_ROOT="/vcpkg" - env.SCILLA_REPO_ROOT="/scilla/0" - stage('Update source code') { - sh "mkdir -p /scilla/0 && cp -r ${WORKSPACE}/* /scilla/0 && ln -s ${WORKSPACE}/.git /scilla/0/.git" - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes" - sh "apt update && apt install -y sudo" - sh "./scripts/install_shellcheck_ubuntu.sh" - } - stage('Test') { - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test" - } - stage('Test server') { - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server" - } - stage('Coveralls') { - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${BUILD_NUMBER}" - } - stage('Lint') { - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint" - } - } - } catch (err) { - if (err.getMessage() == skipciMessage) - currentBuild.result = 'SUCCESS' - else - throw err - } - } - } - } - } -} From 62ea87d97b5845e992466a5519bdc40301fc77e4 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 13:05:25 +0400 Subject: [PATCH 11/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- docker/Dockerfile.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index de62613ee..ee7b96f86 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,14 +1,14 @@ # Requires a Scilla local tag build from docker/Dockerfile -FROM scilla:local +FROM 648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 ENV VCPKG_ROOT="/vcpkg" ENV SCILLA_REPO_ROOT="/scilla/0" +COPY . /scilla/0/ + RUN apt-get update -y \ && apt install -y sudo -COPY / /scilla/0/ - RUN cd /scilla/0 \ && eval $(opam env) \ && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes From 8a22b8a559b5a2a2e75a1279f7bc21856f15d655 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 13:06:57 +0400 Subject: [PATCH 12/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- docker/Dockerfile.test | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index ee7b96f86..2bf8d28ea 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,4 +1,3 @@ -# Requires a Scilla local tag build from docker/Dockerfile FROM 648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 ENV VCPKG_ROOT="/vcpkg" From 540d715c405a87001b3c20bd0fe64635b7dd5a93 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 13:07:42 +0400 Subject: [PATCH 13/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index f04c4e0f6..5f525787a 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -28,9 +28,18 @@ jobs: with: fetch-depth: 0 ref: ${{ inputs.commitOrTag }} + - name: Configure AWS Credentials + uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + - name: Login to the registry + uses: docker/login-action@v2 + with: + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com - name: Build Docker images run: | - DOCKER_BUILDKIT=1 docker build -t scilla:local -f docker/Dockerfile . DOCKER_BUILDKIT=1 docker build -t scilla:tests -f docker/Dockerfile.test . shell: bash - name: Run make test From c641c7acccedd9eb22a8e6027622bea1a0189cf5 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 13:12:11 +0400 Subject: [PATCH 14/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 5f525787a..07f3c27e4 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -13,6 +13,9 @@ on: jobs: run-tests: + permissions: + id-token: read + contents: read name: tests runs-on: docker steps: From 4e974a961accac54f1a8a2fdde35b13575030aac Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 13:13:07 +0400 Subject: [PATCH 15/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 07f3c27e4..d5e91d9da 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -14,8 +14,8 @@ on: jobs: run-tests: permissions: - id-token: read - contents: read + id-token: write + contents: write name: tests runs-on: docker steps: From 6fd28671f790a71b9875d052aa5495173b8b9454 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 13:16:51 +0400 Subject: [PATCH 16/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 78d5928ec..5d3bfef8e 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -42,10 +42,10 @@ jobs: run: echo "tag=v$(tail -n +2 VERSION | head -n1).$(tail -n +4 VERSION | head -n1).$(tail -n +6 VERSION | head -n1)" >> $GITHUB_OUTPUT shell: bash - name: Build Docker images - run: DOCKER_BUILDKIT=1 docker build -t ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.comscilla:${{ steps.set-tag.outputs.tag }} -t zilliqa/scilla:${{ steps.set-tag.outputs.tag }} -f docker/Dockerfile . + run: DOCKER_BUILDKIT=1 docker build -t ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} -t zilliqa/scilla:${{ steps.set-tag.outputs.tag }} -f docker/Dockerfile . shell: bash - name: Push Docker images - run: docker push ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.comscilla:${{ steps.set-tag.outputs.tag }} + run: docker push ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} shell: bash - name: Login to the DockerHub uses: docker/login-action@v2 From 97101237281b305cbe30dc15dddc1a86fabc33d5 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 13:20:11 +0400 Subject: [PATCH 17/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 6 +++--- docker/Dockerfile.test | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index d5e91d9da..e0f7e4d9f 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -47,13 +47,13 @@ jobs: shell: bash - name: Run make test run: | - docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' shell: bash - name: Run make test_server run: | - docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' shell: bash - name: Run make lint run: | - docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' shell: bash diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 2bf8d28ea..a7b305130 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -3,13 +3,13 @@ FROM 648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 ENV VCPKG_ROOT="/vcpkg" ENV SCILLA_REPO_ROOT="/scilla/0" +WORKDIR /scilla/0/ COPY . /scilla/0/ RUN apt-get update -y \ && apt install -y sudo -RUN cd /scilla/0 \ - && eval $(opam env) \ +RUN eval $(opam env) \ && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes RUN ./scripts/install_shellcheck_ubuntu.sh From 035352c5f30036861c67b6066ee8e0369ec26996 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 13:28:24 +0400 Subject: [PATCH 18/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- src/base/cpp/config/discover.ml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/base/cpp/config/discover.ml b/src/base/cpp/config/discover.ml index 5bae54e6d..c50227f51 100644 --- a/src/base/cpp/config/discover.ml +++ b/src/base/cpp/config/discover.ml @@ -30,7 +30,9 @@ let () = We rely on a simple python program (in vcpkg_cmd) to do so portably. *) let vcpkg_cmd = - "python3 -c 'import os, sys; print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]" ^^ "+\"/%s\"))'" + "python3 -c 'import os, sys; \ + print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]" + ^^ "+\"/%s\"))'" in let vcpkg_triplet_path = input_line From cb9e2251c833f31b1918a4fad7ee4eb9760d9f4a Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 15:29:05 +0400 Subject: [PATCH 19/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index e0f7e4d9f..94f501ef2 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -53,6 +53,10 @@ jobs: run: | docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' shell: bash + - name: Run make coveralls + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls GITHUB_RUN_NUMBER=${{ github.run_number }}' + shell: bash - name: Run make lint run: | docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' From fdcddd719c872229e87e260868cfc5162c3b1fdc Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 15:33:25 +0400 Subject: [PATCH 20/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ce5496ef2..b2992f107 100644 --- a/Makefile +++ b/Makefile @@ -232,7 +232,7 @@ coveralls: BISECT_ENABLE=YES make dune build @install ulimit -n 1024; dune exec -- tests/testsuite.exe - bisect-ppx-report coveralls coverage.json --ignore-missing-files --service-name jenkins --service-job-id ${TRAVIS_JOB_ID} + bisect-ppx-report coveralls coverage.json --ignore-missing-files --service-name jenkins --service-job-id ${GITHUB_RUN_NUMBER} curl -L -F json_file=@./coverage.json https://coveralls.io/api/v1/jobs make clean -find . -type f -name 'bisect*.coverage' | xargs rm From 7ddf4e4d49dd151681ee1fdc597b83e4a19f4d94 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 15:51:22 +0400 Subject: [PATCH 21/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- Jenkinsfile | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..ac7cfc28c --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,76 @@ +def podDefinition = """ +apiVersion: "v1" +kind: "Pod" +spec: + nodeSelector: + beta.kubernetes.io/os: "linux" + containers: + - name: "ubuntu" + image: "648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9" + command: + - cat + tty: true + resources: + requests: + ephemeral-storage: "20Gi" + limits: + ephemeral-storage: "20Gi" +""" + +String skipciMessage = 'Aborting because commit message contains [skip ci]' + +timestamps { + ansiColor('gnome-terminal') { + podTemplate(yaml: podDefinition) { + timeout(time: 60, unit: 'MINUTES') { + node(POD_LABEL) { + try { + stage('Checkout scm') { + checkout scm + def pr_skipci = "0" + try { + if (env.CHANGE_TITLE != null && env.CHANGE_TITLE != "") { + pr_skipci = sh(script: "echo ${env.CHANGE_TITLE.replace("(","").replace(")","")} | fgrep -ie '[skip ci]' -e '[ci skip]' | wc -l", returnStdout: true).trim() + } + } catch (err) { + println err.getMessage() + error("Error reading the Pull Request title, please check and eventually remove special characters") + } + def skipci = sh(script: "git log -1 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' | wc -l", returnStdout: true).trim() + if (skipci != "0" || pr_skipci != "0") { + error(skipciMessage) + } + } + container('ubuntu') { + env.VCPKG_ROOT="/vcpkg" + env.SCILLA_REPO_ROOT="/scilla/0" + stage('Update source code') { + sh "mkdir -p /scilla/0 && cp -r ${WORKSPACE}/* /scilla/0 && ln -s ${WORKSPACE}/.git /scilla/0/.git" + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes" + sh "apt update && apt install -y sudo" + sh "./scripts/install_shellcheck_ubuntu.sh" + } + stage('Test') { + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test" + } + stage('Test server') { + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server" + } + stage('Coveralls') { + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${BUILD_NUMBER}" + } + stage('Lint') { + sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint" + } + } + } catch (err) { + if (err.getMessage() == skipciMessage) + currentBuild.result = 'SUCCESS' + else + throw err + } + } + } + } + } +} From d2f7c75d0238d5a870a3982edf2c1a570da493e6 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 15:56:22 +0400 Subject: [PATCH 22/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- src/base/cpp/config/discover.ml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/base/cpp/config/discover.ml b/src/base/cpp/config/discover.ml index c50227f51..5bae54e6d 100644 --- a/src/base/cpp/config/discover.ml +++ b/src/base/cpp/config/discover.ml @@ -30,9 +30,7 @@ let () = We rely on a simple python program (in vcpkg_cmd) to do so portably. *) let vcpkg_cmd = - "python3 -c 'import os, sys; \ - print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]" - ^^ "+\"/%s\"))'" + "python3 -c 'import os, sys; print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]" ^^ "+\"/%s\"))'" in let vcpkg_triplet_path = input_line From 1a0d01a72b6f1f01f8e564b78d210a83ed13a3a9 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 16:04:45 +0400 Subject: [PATCH 23/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 2 +- Makefile | 2 +- src/base/cpp/config/discover.ml | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 94f501ef2..928f90212 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -55,7 +55,7 @@ jobs: shell: bash - name: Run make coveralls run: | - docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls GITHUB_RUN_NUMBER=${{ github.run_number }}' + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}' shell: bash - name: Run make lint run: | diff --git a/Makefile b/Makefile index b2992f107..ce5496ef2 100644 --- a/Makefile +++ b/Makefile @@ -232,7 +232,7 @@ coveralls: BISECT_ENABLE=YES make dune build @install ulimit -n 1024; dune exec -- tests/testsuite.exe - bisect-ppx-report coveralls coverage.json --ignore-missing-files --service-name jenkins --service-job-id ${GITHUB_RUN_NUMBER} + bisect-ppx-report coveralls coverage.json --ignore-missing-files --service-name jenkins --service-job-id ${TRAVIS_JOB_ID} curl -L -F json_file=@./coverage.json https://coveralls.io/api/v1/jobs make clean -find . -type f -name 'bisect*.coverage' | xargs rm diff --git a/src/base/cpp/config/discover.ml b/src/base/cpp/config/discover.ml index 5bae54e6d..c50227f51 100644 --- a/src/base/cpp/config/discover.ml +++ b/src/base/cpp/config/discover.ml @@ -30,7 +30,9 @@ let () = We rely on a simple python program (in vcpkg_cmd) to do so portably. *) let vcpkg_cmd = - "python3 -c 'import os, sys; print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]" ^^ "+\"/%s\"))'" + "python3 -c 'import os, sys; \ + print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]" + ^^ "+\"/%s\"))'" in let vcpkg_triplet_path = input_line From d14122a481d37f0aac7113076b6c32d82fb8e1f1 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Mon, 19 Jun 2023 16:09:38 +0400 Subject: [PATCH 24/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 928f90212..554133ead 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -47,17 +47,17 @@ jobs: shell: bash - name: Run make test run: | - docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' + docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' shell: bash - name: Run make test_server run: | - docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' + docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' shell: bash - name: Run make coveralls run: | - docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}' + docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}' shell: bash - name: Run make lint run: | - docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' + docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' shell: bash From 8e7039ff708df339f66d3e2bf1e43c17b39bc7dc Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 14:34:17 +0400 Subject: [PATCH 25/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- Makefile | 1 - docker/Dockerfile.test | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ce5496ef2..4ae0e7512 100644 --- a/Makefile +++ b/Makefile @@ -226,7 +226,6 @@ coverage : .PHONY : coveralls coveralls: - make clean mkdir -p _build/coverage ./scripts/build_deps.sh BISECT_ENABLE=YES make diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index a7b305130..b8c1f38eb 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -3,13 +3,13 @@ FROM 648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 ENV VCPKG_ROOT="/vcpkg" ENV SCILLA_REPO_ROOT="/scilla/0" -WORKDIR /scilla/0/ COPY . /scilla/0/ -RUN apt-get update -y \ +RUN apt update \ && apt install -y sudo -RUN eval $(opam env) \ +RUN cd /scilla/0 \ + && eval $(opam env) \ && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes -RUN ./scripts/install_shellcheck_ubuntu.sh +RUN /scilla/0/scripts/install_shellcheck_ubuntu.sh From f929448efa37962cbf843bf96a0ee895ec570d0e Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 15:06:04 +0400 Subject: [PATCH 26/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-test.yml | 8 ++++---- Makefile | 3 ++- docker/Dockerfile.test | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 554133ead..928f90212 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -47,17 +47,17 @@ jobs: shell: bash - name: Run make test run: | - docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' shell: bash - name: Run make test_server run: | - docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' shell: bash - name: Run make coveralls run: | - docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}' + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}' shell: bash - name: Run make lint run: | - docker run --rm -i scilla:tests bash -c 'cd /scilla/0 && eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' shell: bash diff --git a/Makefile b/Makefile index 4ae0e7512..fb867a46a 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ clean: dune clean # Remove remaining files/folders ignored by git as defined in .gitignore (-X) # but keeping a local opam switch and other dependencies built. - git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed + git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed --exclude=\!vcpkg_installed/** # Clean up libff installation cleanall: clean @@ -226,6 +226,7 @@ coverage : .PHONY : coveralls coveralls: + make clean mkdir -p _build/coverage ./scripts/build_deps.sh BISECT_ENABLE=YES make diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index b8c1f38eb..60e4a3181 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -3,13 +3,13 @@ FROM 648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 ENV VCPKG_ROOT="/vcpkg" ENV SCILLA_REPO_ROOT="/scilla/0" +WORKDIR /scilla/0/ COPY . /scilla/0/ RUN apt update \ && apt install -y sudo -RUN cd /scilla/0 \ - && eval $(opam env) \ +RUN eval $(opam env) \ && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes -RUN /scilla/0/scripts/install_shellcheck_ubuntu.sh +RUN ./scripts/install_shellcheck_ubuntu.sh From c4c217c2ca1e90cd6f5567addd5c62a8cbcce069 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 15:07:28 +0400 Subject: [PATCH 27/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- Jenkinsfile | 76 ----------------------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index ac7cfc28c..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,76 +0,0 @@ -def podDefinition = """ -apiVersion: "v1" -kind: "Pod" -spec: - nodeSelector: - beta.kubernetes.io/os: "linux" - containers: - - name: "ubuntu" - image: "648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9" - command: - - cat - tty: true - resources: - requests: - ephemeral-storage: "20Gi" - limits: - ephemeral-storage: "20Gi" -""" - -String skipciMessage = 'Aborting because commit message contains [skip ci]' - -timestamps { - ansiColor('gnome-terminal') { - podTemplate(yaml: podDefinition) { - timeout(time: 60, unit: 'MINUTES') { - node(POD_LABEL) { - try { - stage('Checkout scm') { - checkout scm - def pr_skipci = "0" - try { - if (env.CHANGE_TITLE != null && env.CHANGE_TITLE != "") { - pr_skipci = sh(script: "echo ${env.CHANGE_TITLE.replace("(","").replace(")","")} | fgrep -ie '[skip ci]' -e '[ci skip]' | wc -l", returnStdout: true).trim() - } - } catch (err) { - println err.getMessage() - error("Error reading the Pull Request title, please check and eventually remove special characters") - } - def skipci = sh(script: "git log -1 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' | wc -l", returnStdout: true).trim() - if (skipci != "0" || pr_skipci != "0") { - error(skipciMessage) - } - } - container('ubuntu') { - env.VCPKG_ROOT="/vcpkg" - env.SCILLA_REPO_ROOT="/scilla/0" - stage('Update source code') { - sh "mkdir -p /scilla/0 && cp -r ${WORKSPACE}/* /scilla/0 && ln -s ${WORKSPACE}/.git /scilla/0/.git" - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes" - sh "apt update && apt install -y sudo" - sh "./scripts/install_shellcheck_ubuntu.sh" - } - stage('Test') { - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test" - } - stage('Test server') { - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server" - } - stage('Coveralls') { - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${BUILD_NUMBER}" - } - stage('Lint') { - sh "cd /scilla/0 && eval \$(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint" - } - } - } catch (err) { - if (err.getMessage() == skipciMessage) - currentBuild.result = 'SUCCESS' - else - throw err - } - } - } - } - } -} From 2c7ad5888c7b8020ef84f0af8b0f6d639e6b9423 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 15:57:30 +0400 Subject: [PATCH 28/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-dev.yml | 27 ++++++--------- .github/workflows/ci-image-release.yml | 46 ++++++++++++-------------- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci-image-dev.yml b/.github/workflows/ci-image-dev.yml index 96bea9cb5..da978e0b0 100644 --- a/.github/workflows/ci-image-dev.yml +++ b/.github/workflows/ci-image-dev.yml @@ -31,23 +31,16 @@ jobs: with: fetch-depth: 0 ref: ${{ inputs.commitOrTag }} - - name: Configure AWS Credentials - uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1 + - name: Docker build and push + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: + file: docker/Dockerfile + push: true + tag: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla + tag-length: 7 + tag-latest: ${{ github.ref == 'refs/heads/${{ github.event.repository.default_branch }}' }} + registry: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + aws-region: ${{ secrets.AWS_REGION }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} oidc-role: ${{ secrets.OIDC_ROLE }} - aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} - - name: Login to the registry - uses: docker/login-action@v2 - with: - registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com - - name: Image tag - id: set-tag - run: echo "tag=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - shell: bash - - name: Build Docker images - run: DOCKER_BUILDKIT=1 docker build -t ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} -f docker/Dockerfile . - shell: bash - - name: Push Docker images - run: docker push ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} - shell: bash + cache-key: ${{ github.event.repository.name }} diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 5d3bfef8e..556165840 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -27,31 +27,29 @@ jobs: with: fetch-depth: 0 ref: ${{ inputs.commitOrTag }} - - name: Configure AWS Credentials - uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1 + - name: Docker build and push + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: + file: docker/Dockerfile + push: true + tag: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla + tag-length: 7 + tag-latest: ${{ github.ref == 'refs/heads/${{ github.event.repository.default_branch }}' }} + registry: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + aws-region: ${{ secrets.AWS_REGION }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} oidc-role: ${{ secrets.OIDC_ROLE }} - aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} - - name: Login to the registry - uses: docker/login-action@v2 + cache-key: ${{ github.event.repository.name }} + - name: Docker build and push (Dockerhub) + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: - registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com - - name: Image tag - id: set-tag - run: echo "tag=v$(tail -n +2 VERSION | head -n1).$(tail -n +4 VERSION | head -n1).$(tail -n +6 VERSION | head -n1)" >> $GITHUB_OUTPUT - shell: bash - - name: Build Docker images - run: DOCKER_BUILDKIT=1 docker build -t ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} -t zilliqa/scilla:${{ steps.set-tag.outputs.tag }} -f docker/Dockerfile . - shell: bash - - name: Push Docker images - run: docker push ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} - shell: bash - - name: Login to the DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Push Docker images - run: docker push zilliqa/scilla:${{ steps.set-tag.outputs.tag }} - shell: bash + file: .github/test/Dockerfile + push: true + tag: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla:dockerhub + registry: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + cache-key: ${{ github.event.repository.name }} + aws-region: ${{ secrets.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ecr-read-write + oidc-role: ${{ secrets.OIDC_ROLE }} + registry-username: ${{ secrets.DOCKERHUB_USERNAME }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} From 8c60ad8ff2e2f3a037a8a6001c0d618d349054f2 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 16:20:33 +0400 Subject: [PATCH 29/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-dev.yml | 6 +++--- .github/workflows/ci-image-release.yml | 10 +++++----- .github/workflows/ci-image-test.yml | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-image-dev.yml b/.github/workflows/ci-image-dev.yml index da978e0b0..5239ad8e6 100644 --- a/.github/workflows/ci-image-dev.yml +++ b/.github/workflows/ci-image-dev.yml @@ -36,10 +36,10 @@ jobs: with: file: docker/Dockerfile push: true - tag: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla - tag-length: 7 + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla + tag-length: 8 tag-latest: ${{ github.ref == 'refs/heads/${{ github.event.repository.default_branch }}' }} - registry: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com aws-region: ${{ secrets.AWS_REGION }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} oidc-role: ${{ secrets.OIDC_ROLE }} diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 556165840..62cec11a7 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -32,10 +32,10 @@ jobs: with: file: docker/Dockerfile push: true - tag: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla tag-length: 7 tag-latest: ${{ github.ref == 'refs/heads/${{ github.event.repository.default_branch }}' }} - registry: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com aws-region: ${{ secrets.AWS_REGION }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} oidc-role: ${{ secrets.OIDC_ROLE }} @@ -45,11 +45,11 @@ jobs: with: file: .github/test/Dockerfile push: true - tag: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla:dockerhub - registry: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla:dockerhub + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com cache-key: ${{ github.event.repository.name }} aws-region: ${{ secrets.AWS_REGION }} - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ecr-read-write + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}:role/ecr-read-write oidc-role: ${{ secrets.OIDC_ROLE }} registry-username: ${{ secrets.DOCKERHUB_USERNAME }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 928f90212..2a3e53a55 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -36,11 +36,11 @@ jobs: with: role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} oidc-role: ${{ secrets.OIDC_ROLE }} - aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + aws-region: ${{ secrets.AWS_REGION }} - name: Login to the registry uses: docker/login-action@v2 with: - registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com - name: Build Docker images run: | DOCKER_BUILDKIT=1 docker build -t scilla:tests -f docker/Dockerfile.test . From 999d115b3f20c17c8b45e0057cee950d40a075c6 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 16:22:22 +0400 Subject: [PATCH 30/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-dev.yml | 2 +- .github/workflows/ci-image-release.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-image-dev.yml b/.github/workflows/ci-image-dev.yml index 5239ad8e6..68a9d43d9 100644 --- a/.github/workflows/ci-image-dev.yml +++ b/.github/workflows/ci-image-dev.yml @@ -38,7 +38,7 @@ jobs: push: true tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla tag-length: 8 - tag-latest: ${{ github.ref == 'refs/heads/${{ github.event.repository.default_branch }}' }} + tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com aws-region: ${{ secrets.AWS_REGION }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 62cec11a7..8b995e11b 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -33,8 +33,8 @@ jobs: file: docker/Dockerfile push: true tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla - tag-length: 7 - tag-latest: ${{ github.ref == 'refs/heads/${{ github.event.repository.default_branch }}' }} + tag-length: 8 + tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com aws-region: ${{ secrets.AWS_REGION }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} From 5326f3a7ccd07e6667d1843082919401f32a8ef2 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 16:35:33 +0400 Subject: [PATCH 31/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-dev.yml | 6 +++--- .github/workflows/ci-image-release.yml | 16 +++++++--------- .github/workflows/ci-image-test.yml | 4 ++-- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-image-dev.yml b/.github/workflows/ci-image-dev.yml index 68a9d43d9..3188f694c 100644 --- a/.github/workflows/ci-image-dev.yml +++ b/.github/workflows/ci-image-dev.yml @@ -36,11 +36,11 @@ jobs: with: file: docker/Dockerfile push: true - tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla tag-length: 8 tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} - registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com - aws-region: ${{ secrets.AWS_REGION }} + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} oidc-role: ${{ secrets.OIDC_ROLE }} cache-key: ${{ github.event.repository.name }} diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 8b995e11b..f262f8c89 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -32,24 +32,22 @@ jobs: with: file: docker/Dockerfile push: true - tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla tag-length: 8 tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} - registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com - aws-region: ${{ secrets.AWS_REGION }} + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} oidc-role: ${{ secrets.OIDC_ROLE }} cache-key: ${{ github.event.repository.name }} - name: Docker build and push (Dockerhub) uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: - file: .github/test/Dockerfile + file: docker/Dockerfile push: true - tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/scilla:dockerhub - registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + tag: Zilliqa/scilla + tag-length: 8 + tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} cache-key: ${{ github.event.repository.name }} - aws-region: ${{ secrets.AWS_REGION }} - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}:role/ecr-read-write - oidc-role: ${{ secrets.OIDC_ROLE }} registry-username: ${{ secrets.DOCKERHUB_USERNAME }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 2a3e53a55..928f90212 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -36,11 +36,11 @@ jobs: with: role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} oidc-role: ${{ secrets.OIDC_ROLE }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} - name: Login to the registry uses: docker/login-action@v2 with: - registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com - name: Build Docker images run: | DOCKER_BUILDKIT=1 docker build -t scilla:tests -f docker/Dockerfile.test . From d7ecac94c7850ccf5ff98aa9b761b53b785b21cb Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 17:22:15 +0400 Subject: [PATCH 32/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-dev.yml | 1 - .github/workflows/ci-image-test.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-image-dev.yml b/.github/workflows/ci-image-dev.yml index 3188f694c..4ac3ab237 100644 --- a/.github/workflows/ci-image-dev.yml +++ b/.github/workflows/ci-image-dev.yml @@ -38,7 +38,6 @@ jobs: push: true tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla tag-length: 8 - tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 928f90212..b7ad91119 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -43,7 +43,7 @@ jobs: registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com - name: Build Docker images run: | - DOCKER_BUILDKIT=1 docker build -t scilla:tests -f docker/Dockerfile.test . + DOCKER_BUILDKIT=1 docker build -t scilla:tests --build-arg ACCOUNT_ID=${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }} -f docker/Dockerfile.test . shell: bash - name: Run make test run: | From c10bfa208bf2caf15a5d0f87a374dfa7b3957f9c Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 17:23:53 +0400 Subject: [PATCH 33/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- docker/Dockerfile.test | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 60e4a3181..f1e533cb7 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,4 +1,6 @@ -FROM 648273915458.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 +ARG ACCOUNT_ID + +FROM ${ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 ENV VCPKG_ROOT="/vcpkg" ENV SCILLA_REPO_ROOT="/scilla/0" From db56f931b8ac92b430d16d47b3a37d596b41626f Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 17:30:35 +0400 Subject: [PATCH 34/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index f262f8c89..8f6ed1bb3 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -1,4 +1,4 @@ -scillaname: CI - Release +name: CI - Release on: workflow_dispatch: @@ -8,12 +8,7 @@ on: required: false default: '' push: - branches: - - 'release/**' - paths: - - 'VERSION' tags: - - v* jobs: release-image: From 92996ebbaf44edee47d3bd34d1657ee86f00abe2 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 17:55:47 +0400 Subject: [PATCH 35/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 8f6ed1bb3..b7adfa0cd 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -17,19 +17,26 @@ jobs: contents: write runs-on: docker steps: - - name: 'Checkout scm ${{ inputs.commitOrTag }}' + - name: 'Checkout scm' uses: actions/checkout@v3 with: fetch-depth: 0 - ref: ${{ inputs.commitOrTag }} + - name: Check if the tag matches the master commit + id: check-latest + run: | + if test $(git rev-parse origin/master) = $(git rev-parse HEAD); then + echo "latest=true" >> $GITHUB_OUTPUT + else + echo "latest=false" >> $GITHUB_OUTPUT + fi + shell: bash - name: Docker build and push uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: file: docker/Dockerfile push: true - tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla - tag-length: 8 - tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ github.ref_name }} + tag-latest: ${{ steps.check-latest.outputs.latest }} registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} @@ -40,8 +47,7 @@ jobs: with: file: docker/Dockerfile push: true - tag: Zilliqa/scilla - tag-length: 8 + tag: Zilliqa/scilla:${{ github.ref_name }} tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} cache-key: ${{ github.event.repository.name }} registry-username: ${{ secrets.DOCKERHUB_USERNAME }} From b534d452be5202749b57453f4222d99cd8fae480 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 17:57:34 +0400 Subject: [PATCH 36/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index b7adfa0cd..ca5d665c4 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Check if the tag matches the master commit + - name: Check if the tag head is the same as master head id: check-latest run: | if test $(git rev-parse origin/master) = $(git rev-parse HEAD); then @@ -46,7 +46,7 @@ jobs: uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: file: docker/Dockerfile - push: true + push: false tag: Zilliqa/scilla:${{ github.ref_name }} tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} cache-key: ${{ github.event.repository.name }} From b2805891fd2c676c99d71f37f059dfbfc9ef11d9 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 18:02:09 +0400 Subject: [PATCH 37/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index ca5d665c4..2e127adbf 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -24,7 +24,7 @@ jobs: - name: Check if the tag head is the same as master head id: check-latest run: | - if test $(git rev-parse origin/master) = $(git rev-parse HEAD); then + if test $(git rev-parse origin/devops-700) = $(git rev-parse HEAD); then echo "latest=true" >> $GITHUB_OUTPUT else echo "latest=false" >> $GITHUB_OUTPUT From f1c48c5ab42a71ccf82d877af99d070397a6a0cb Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 18:10:41 +0400 Subject: [PATCH 38/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 2e127adbf..6620e2c33 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -1,12 +1,6 @@ name: CI - Release on: - workflow_dispatch: - inputs: - commitOrTag: - description: 'Commit or tag' - required: false - default: '' push: tags: @@ -24,7 +18,7 @@ jobs: - name: Check if the tag head is the same as master head id: check-latest run: | - if test $(git rev-parse origin/devops-700) = $(git rev-parse HEAD); then + if test $(git rev-parse origin/master) = $(git rev-parse HEAD); then echo "latest=true" >> $GITHUB_OUTPUT else echo "latest=false" >> $GITHUB_OUTPUT @@ -46,9 +40,9 @@ jobs: uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: file: docker/Dockerfile - push: false + push: true tag: Zilliqa/scilla:${{ github.ref_name }} - tag-latest: ${{ github.ref_name == github.event.repository.default_branch }} + tag-latest: ${{ steps.check-latest.outputs.latest }} cache-key: ${{ github.event.repository.name }} registry-username: ${{ secrets.DOCKERHUB_USERNAME }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} From 0b430639c0c32702dc497bdae8585e4d266362ee Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 18:20:38 +0400 Subject: [PATCH 39/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 6620e2c33..fb8728c4a 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -1,6 +1,12 @@ name: CI - Release on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' push: tags: @@ -11,12 +17,23 @@ jobs: contents: write runs-on: docker steps: - - name: 'Checkout scm' + - name: 'Checkout scm ${{ inputs.commitOrTag }}' uses: actions/checkout@v3 with: fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Check if the commit or tag was passed manually + id: set-tag + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "tag=$(git rev-parse HEAD | cut -c1-8)" >> $GITHUB_OUTPUT + else + echo "tag=${{ github.ref_name }}" >> $GITHUB_OUTPUT + fi + shell: bash - name: Check if the tag head is the same as master head id: check-latest + if: github.event_name == 'workflow_dispatch' run: | if test $(git rev-parse origin/master) = $(git rev-parse HEAD); then echo "latest=true" >> $GITHUB_OUTPUT @@ -29,7 +46,7 @@ jobs: with: file: docker/Dockerfile push: true - tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ github.ref_name }} + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} tag-latest: ${{ steps.check-latest.outputs.latest }} registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} @@ -41,7 +58,7 @@ jobs: with: file: docker/Dockerfile push: true - tag: Zilliqa/scilla:${{ github.ref_name }} + tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }} tag-latest: ${{ steps.check-latest.outputs.latest }} cache-key: ${{ github.event.repository.name }} registry-username: ${{ secrets.DOCKERHUB_USERNAME }} From cb65bcda5ebf6325f10bdcd5b800c99ba77c9d25 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 18:25:10 +0400 Subject: [PATCH 40/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index fb8728c4a..1a0d2cdaf 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -57,7 +57,7 @@ jobs: uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: file: docker/Dockerfile - push: true + push: false tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }} tag-latest: ${{ steps.check-latest.outputs.latest }} cache-key: ${{ github.event.repository.name }} From d594780af2ad7dc8e534b30c96270d8bdf787231 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 18:30:05 +0400 Subject: [PATCH 41/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 1a0d2cdaf..792210074 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -33,7 +33,7 @@ jobs: shell: bash - name: Check if the tag head is the same as master head id: check-latest - if: github.event_name == 'workflow_dispatch' + if: github.event_name != 'workflow_dispatch' run: | if test $(git rev-parse origin/master) = $(git rev-parse HEAD); then echo "latest=true" >> $GITHUB_OUTPUT From 49704bdcd30395454b4019a7ad323ff0aaed910a Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 18:36:05 +0400 Subject: [PATCH 42/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 792210074..50d513426 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -57,7 +57,7 @@ jobs: uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: file: docker/Dockerfile - push: false + push: true tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }} tag-latest: ${{ steps.check-latest.outputs.latest }} cache-key: ${{ github.event.repository.name }} From e632e8cff7a328f6c78273233b12b16ef0ad584c Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 18:38:42 +0400 Subject: [PATCH 43/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 50d513426..792210074 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -57,7 +57,7 @@ jobs: uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: file: docker/Dockerfile - push: true + push: false tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }} tag-latest: ${{ steps.check-latest.outputs.latest }} cache-key: ${{ github.event.repository.name }} From 58cf4af79fe1aa1278e73454aa877610428b7f70 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 18:42:29 +0400 Subject: [PATCH 44/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index 792210074..b1c766cf7 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -9,6 +9,7 @@ on: default: '' push: tags: + - v* jobs: release-image: From bee145995d425a9fa4c36b7e9b81f6f5a092fc8c Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Tue, 20 Jun 2023 18:45:41 +0400 Subject: [PATCH 45/47] feat: DEVOPS-700 scilla pipelines from jenkins to gha --- .github/workflows/ci-image-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml index b1c766cf7..0b3cf335a 100644 --- a/.github/workflows/ci-image-release.yml +++ b/.github/workflows/ci-image-release.yml @@ -58,7 +58,7 @@ jobs: uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 with: file: docker/Dockerfile - push: false + push: true tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }} tag-latest: ${{ steps.check-latest.outputs.latest }} cache-key: ${{ github.event.repository.name }} From 6d71194d1ed1a121482dcc94fb8eda0050c90921 Mon Sep 17 00:00:00 2001 From: n-hutton Date: Thu, 18 Jan 2024 20:07:32 +0000 Subject: [PATCH 46/47] the changes without b64 stuff --- docker/Dockerfile | 14 ++++++++++++-- handle_connection.sh | 27 +++++++++++++++++++++++++++ run_scilla_tcp.sh | 21 +++++++++++++++++++++ write_files.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100755 handle_connection.sh create mode 100755 run_scilla_tcp.sh create mode 100755 write_files.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 42c1671b1..e2f166de6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,10 @@ # escape=\ # Common dependencies of the builder and runner stages. -FROM ubuntu:22.04 AS base +FROM --platform=linux/amd64 ubuntu:22.04 AS base + +RUN apt-get update -y + # Format guideline: one package per line and keep them alphabetically sorted RUN apt-get update -y \ && apt-get install -y software-properties-common \ @@ -92,6 +95,9 @@ COPY scilla.opam . COPY shell.nix . COPY .ocamlformat . COPY tests/ tests +COPY handle_connection.sh . +COPY run_scilla_tcp.sh . +COPY write_files.py . # Make sure vcpkg installs brings in the dependencies RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic @@ -115,10 +121,14 @@ RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \ && rm -rf vcpkg_installed \ && ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed -FROM ubuntu:22.04 +FROM --platform=linux/amd64 ubuntu:22.04 RUN apt-get update -y \ && apt-get install -y build-essential \ + netcat-openbsd \ + vim \ + socat \ + python3 \ libgmp-dev ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}" diff --git a/handle_connection.sh b/handle_connection.sh new file mode 100755 index 000000000..55b5f5e3d --- /dev/null +++ b/handle_connection.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +echo "Starting server." 1>&2 + +# Start scilla-server in the background +rm -rf /tmp/scilla-socket/server.sock +/scilla/0/bin/scilla-server -socket /tmp/scilla-socket/server.sock 1>&2 & + +# Get the PID of scilla-server +SERVER_PID=$! + +echo "forwarding traffic." 1>&2 + +# Block until we see the scilla server has provided the socket +while [ ! -S /tmp/scilla-socket/server.sock ]; do sleep 1; done + +echo "forwarding other traffic." 1>&2 + +# Forward traffic from stdin (provided by socat) to Unix socket. +# Note this means we ONLY use stderr when echoing within this script to avoid it going to the socket +socat - UNIX-CONNECT:/tmp/scilla-socket/server.sock + +echo "connection closed - killing" 1>&2 +echo "" 1>&2 + +# Kill the scilla-server process when done +kill $SERVER_PID diff --git a/run_scilla_tcp.sh b/run_scilla_tcp.sh new file mode 100755 index 000000000..a9eca4632 --- /dev/null +++ b/run_scilla_tcp.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +echo "Starting server." 1>&2 + +rm -rf myfifo 1>&2 +mkfifo myfifo 1>&2 + +# Clear the directories we will be using the files for +mkdir -p /tmp/scilla_init/ +mkdir -p /tmp/scilla_input/ + +rm -rf /tmp/scilla_init/* +rm -rf /tmp/scilla_input/* + +# netcat listen to 12346 and forward it to/from a local socket (this is the state queries/updates) +nc -lkv 12346 myfifo & + + +# socat - every connection on 12345, run the script handle_connection and pipe the input to it (this is the initial run/create/etc commands) +socat TCP-LISTEN:12347,reuseaddr,fork EXEC:"/scilla/0/write_files.py",nofork & +socat TCP-LISTEN:12345,reuseaddr,fork EXEC:"/scilla/0/handle_connection.sh",nofork diff --git a/write_files.py b/write_files.py new file mode 100755 index 000000000..520afd0d8 --- /dev/null +++ b/write_files.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import sys +import os + +def main(): + # Read the input as binary data from stdin + input_data = sys.stdin.buffer.read() + + # Split the input into a path and data + try: + path, data = input_data.split(b' ', 1) + except ValueError: + print("Invalid input format. Please provide a valid absolute path and data.") + sys.exit(1) + + # Ensure the path is absolute + if not os.path.isabs(path.decode()): + print("Please provide an absolute path.") + sys.exit(1) + + # Write binary data to the specified file + try: + with open(path.decode(), 'wb') as file: + file.write(data) + print(f"Data written to {path.decode()}") + except Exception as e: + print(f"Error writing data to {path.decode()}: {str(e)}") + sys.exit(1) + +if __name__ == "__main__": + main() From 0d3db68df445866ec0636eb614b974e7d083e5ac Mon Sep 17 00:00:00 2001 From: n-hutton Date: Thu, 1 Feb 2024 14:48:28 +0000 Subject: [PATCH 47/47] remove space --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e2f166de6..74d0d720d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -121,7 +121,7 @@ RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \ && rm -rf vcpkg_installed \ && ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed -FROM --platform=linux/amd64 ubuntu:22.04 +FROM --platform=linux/amd64 ubuntu:22.04 RUN apt-get update -y \ && apt-get install -y build-essential \