From ab31465c7383c759b1ed1bd38e6212b00fcf8c0d Mon Sep 17 00:00:00 2001 From: Richard Hallett Date: Wed, 7 Feb 2024 11:09:33 +0100 Subject: [PATCH] Rework github workflows for staging deploy Use shared workflows and refactor deployment code. Dropping support for dockerhub images. Test workflow is now shared test workflow. --- .github/workflows/build_docker.yml | 57 +++++++++ .github/workflows/deploy.yml | 180 +++-------------------------- .github/workflows/deployment.yml | 64 ++++++++++ .github/workflows/test.yml | 18 ++- docker-compose.yml | 2 +- 5 files changed, 149 insertions(+), 172 deletions(-) create mode 100644 .github/workflows/build_docker.yml create mode 100644 .github/workflows/deployment.yml diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml new file mode 100644 index 0000000..c0574a0 --- /dev/null +++ b/.github/workflows/build_docker.yml @@ -0,0 +1,57 @@ +name: Build Docker + +on: + workflow_call: + inputs: + image: + required: true + type: string + dockerfile: + required: true + type: string + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }}-${{ inputs.image }}" + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ inputs.image }} + tags: | + type=ref,event=tag + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Push to GitHub Packages + uses: docker/build-push-action@v4 + with: + context: . + file: ${{ inputs.dockerfile }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 40de715..7d50699 100755 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,4 @@ -name: Deploy +name: Deploy to staging on: push: branches: @@ -6,175 +6,21 @@ on: jobs: test: - runs-on: ubuntu-latest - services: - memcached: - image: memcached:1.4.31 - ports: - - 11211/udp - mysql: - image: mysql:5.7 - env: - MYSQL_DATABASE: metrics_stage - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - ports: - - 3306 - redis: - image: redis:6 - ports: - - 6379:6379 - env: - MEMCACHE_SERVERS: "localhost:11211" - MYSQL_HOST: "127.0.0.1" - MYSQL_DATABASE: metrics_stage - MYSQL_USER: root - SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }} - SESSION_ENCRYPTED_COOKIE_SALT: ${{ secrets.SESSION_ENCRYPTED_COOKIE_SALT }} - JWT_PRIVATE_KEY: ${{ secrets.JWT_PRIVATE_KEY }} - JWT_PUBLIC_KEY: ${{ secrets.JWT_PUBLIC_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} - steps: - - uses: actions/checkout@v2 - - name: Set up Ruby 2.6 - uses: actions/setup-ruby@v1 - with: - ruby-version: '2.6.x' - - uses: actions/cache@v2 - with: - path: vendor/bundle - key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }} - restore-keys: | - ${{ runner.os }}-gems- - - name: Install - env: - MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} - run: | - cp .env.build .env - gem install bundler - bundle config path vendor/bundle - bundle install --jobs 4 --retry 3 - sudo /etc/init.d/mysql start - bundle exec rails db:setup RAILS_ENV=test - - name: Lint and Test - env: - MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} - MEMCACHE_SERVERS: "localhost:11211" - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_S3_BUCKET: 'metrics-api.stage.datacite.org' - run: | - # bundle exec rubocop - bundle exec rspec - echo $? - - name: Publish code coverage - uses: paambaati/codeclimate-action@v2.7.5 - env: - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} + uses: ./.github/workflows/test.yml build: needs: test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Push to Docker Hub - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - push: true - tags: ${{ github.repository }}:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - name: Push to GitHub Packages - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - push: true - tags: ghcr.io/${{ github.repository }}:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache + uses: ./.github/workflows/build_docker.yml + with: + image: ghcr.io/${{ github.repository }}web + dockerfile: ./Dockerfile deploy: needs: [test, build] - runs-on: ubuntu-latest - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Extract variables - shell: bash - run: | - echo "::set-output name=BRANCH::$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" - echo "::set-output name=TAG::$(git tag --points-at HEAD)" - echo "::set-output name=GIT_SHA::$(git rev-parse HEAD)" - echo "::set-output name=GIT_SHA_SHORT::$(git rev-parse --short HEAD)" - echo "::set-output name=MESSAGE::$(git log --format=%B -n 1 ${{ github.event.after }})" - id: extract_variables - - - name: Checkout terraform config repo - uses: actions/checkout@v2 - with: - # public repo with terraform configuration - repository: 'datacite/mastino' - persist-credentials: false - - name: Commit changes to terraform config repository - # use go template in terraform config repository to update git sha and tag - # commit and push changes to trigger terraform workflow - run: | - export GIT_SHA=${{ steps.extract_variables.outputs.GIT_SHA_SHORT }} - export GIT_TAG=${{ steps.extract_variables.outputs.GIT_TAG }} - wget https://github.com/jwilder/dockerize/releases/download/v0.6.0/dockerize-linux-amd64-v0.6.0.tar.gz - tar -xzvf dockerize-linux-amd64-v0.6.0.tar.gz - rm dockerize-linux-amd64-v0.6.0.tar.gz - ./dockerize -template stage/services/metrics-api/_sashimi.auto.tfvars.tmpl:stage/services/metrics-api/_sashimi.auto.tfvars - - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add stage/services/metrics-api/_sashimi.auto.tfvars - git commit -m "Adding sashimi git variables for commit ${{ steps.extract_variables.outputs.GIT_SHA }}" - - name: Push changes - uses: ad-m/github-push-action@v0.6.0 - with: - github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - repository: 'datacite/mastino' - branch: 'refs/heads/master' - tags: false - - - name: Notify Slack - uses: edge/simple-slack-notify@master - with: - channel: '#ops' - color: 'good' - text: 'A new version of the is been deployed to stage.' - failure_text: '${env.GITHUB_WORKFLOW} (${env.GITHUB_RUN_NUMBER}) build failed' - fields: | - [{ "title": "Commit message", "value": "${{ steps.extract_variables.outputs.MESSAGE }}" }, - { "title": "Committed by", "value": "", "short": true }, - { "title": "Commit SHA", "value": "", "short": true }, - { "title": "Repository", "value": "", "short": true }, - { "title": "Branch", "value": "", "short": true }] + uses: ./.github/workflows/deployment.yml + with: + versionvar_filename: sashimi + service_name: metrics-api + env_name: stage + secrets: + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml new file mode 100644 index 0000000..4a9556c --- /dev/null +++ b/.github/workflows/deployment.yml @@ -0,0 +1,64 @@ +name: Deploy + +on: + workflow_call: + inputs: + versionvar_filename: + required: true + type: string + service_name: + required: true + type: string + env_name: + required: true + type: string + secrets: + PERSONAL_ACCESS_TOKEN: + required: true + +jobs: + + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Extract variables + shell: bash + run: | + echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT + echo "TAG=$(git tag --points-at HEAD)" >> $GITHUB_OUTPUT + echo "GIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + id: extract_variables + + - name: Checkout terraform config repo + uses: actions/checkout@v3 + with: + repository: 'datacite/mastino' + persist-credentials: false + + - name: Commit changes to terraform config repository + run: | + export GIT_SHA=${{ steps.extract_variables.outputs.GIT_SHA_SHORT }} + export GIT_TAG=${{ steps.extract_variables.outputs.TAG }} + + export VERSION_FILENAME=${{ inputs.env_name }}/services/${{ inputs.service_name }}/${{ inputs.versionvar_filename }}.auto.tfvars + + sed -e "s/{{ .Env.GIT_SHA }}/$GIT_SHA/g" -e "s/{{ .Env.GIT_TAG }}/$GIT_TAG/g" $VERSION_FILENAME.tmpl > $VERSION_FILENAME + + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add $VERSION_FILENAME + git commit -m "Adding ${{ inputs.versionvar_filename }} git variables for commit ${{ steps.extract_variables.outputs.GIT_TAG }}" + + - name: Push changes + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + repository: 'datacite/mastino' + branch: 'refs/heads/master' + tags: false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e0a9de7..5b7b3aa 100755 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: Test workflow -on: workflow_dispatch +on: workflow_call jobs: test: @@ -13,7 +13,7 @@ jobs: mysql: image: mysql:5.7 env: - MYSQL_DATABASE: metrics + MYSQL_DATABASE: metrics_test MYSQL_ALLOW_EMPTY_PASSWORD: "yes" ports: - 3306 @@ -24,7 +24,7 @@ jobs: env: MEMCACHE_SERVERS: "localhost:11211" MYSQL_HOST: "127.0.0.1" - MYSQL_DATABASE: metrics + MYSQL_DATABASE: metrics_test MYSQL_USER: root SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }} SESSION_ENCRYPTED_COOKIE_SALT: ${{ secrets.SESSION_ENCRYPTED_COOKIE_SALT }} @@ -54,4 +54,14 @@ jobs: bundle config path vendor/bundle bundle install --jobs 4 --retry 3 sudo /etc/init.d/mysql start - bundle exec rails db:setup RAILS_ENV=test \ No newline at end of file + bundle exec rails db:setup RAILS_ENV=test + - name: Test + env: + MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} + MEMCACHE_SERVERS: "localhost:11211" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_S3_BUCKET: 'metrics-api.stage.datacite.org' + run: | + bundle exec rspec + echo $? \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 762917c..fb3f3da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '2' services: web: env_file: .env - image: datacite/sashimi + image: ghcr.io/datacite/sashimi ports: - "8075:80" - "2266:22"