From 578dc8d69ca17e64435959b6b9dcd540fc98bd1f Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Thu, 29 Aug 2024 16:16:50 +1000 Subject: [PATCH] Test --- .github/workflows/test_changed_notebooks.yaml | 130 +++++++++++++----- .github/workflows/test_notebooks.yml | 104 +++++++------- 2 files changed, 144 insertions(+), 90 deletions(-) diff --git a/.github/workflows/test_changed_notebooks.yaml b/.github/workflows/test_changed_notebooks.yaml index 5543db7cd..241634a67 100644 --- a/.github/workflows/test_changed_notebooks.yaml +++ b/.github/workflows/test_changed_notebooks.yaml @@ -4,47 +4,101 @@ on: pull_request: branches: [ develop ] +permissions: + id-token: write # Required for OIDC authentication to AWS + contents: read + jobs: test_files: runs-on: ubuntu-latest + + # Automatically activate our mamba environment when it's ready + defaults: + run: + shell: bash -l {0} steps: - - uses: actions/checkout@v3 - # with: - # fetch-depth: 0 - - # - name: Set up Python - # uses: actions/setup-python@v4 - # with: - # python-version: '3.x' - - - name: Get changed notebook files - id: changed-notebooks - uses: tj-actions/changed-files@v44 - with: - files: '**/*.ipynb' - separator: ' ' - - - name: Print changed notebook files - if: steps.changed-notebooks.outputs.any_changed == 'true' - run: | - echo "Changed notebook files:" - echo "${{ steps.changed-notebooks.outputs.all_changed_files }}" - - - name: Get changed Python files - id: changed-py-files - uses: tj-actions/changed-files@v44 - with: - files: '**/*.py' - separator: ' ' - - - name: Print changed Python files - if: steps.changed-py-files.outputs.any_changed == 'true' - run: | - echo "Changed Python files:" - echo "${{ steps.changed-py-files.outputs.all_changed_files }}" + - uses: actions/checkout@v4 + with: + path: dea-notebooks + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly + aws-region: ap-southeast-2 + role-duration-seconds: 7200 # 2 hours + + - name: Copy tide modelling files with the AWS CLI + run: aws s3 sync s3://dea-non-public-data/tide_models/tide_models tide_models + + - name: Login to Amazon ECR Private + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Pull dea-sandbox image from ECR + run: | + docker pull 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox + aws-region: ap-southeast-2 + role-duration-seconds: 7200 # 2 hours + + - name: Get Database Credentials + run: | + username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption --query Parameter.Value --output text) + echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV + + - name: Open Port Forward to RDS + run: | + npx basti connect \ + --custom-target-vpc vpc-086904199e505c1f6 \ + --custom-target-host db-aurora-dea-sandbox-eks-1.cos5zfpkso9m.ap-southeast-2.rds.amazonaws.com \ + --custom-target-port 5432 \ + --local-port 5432 & + npx wait-on --timeout 120000 --interval 1000 tcp:127.0.0.1:5432 + echo "PGPORT=5432" >> $GITHUB_ENV + echo "PGHOST=localhost" >> $GITHUB_ENV + + - name: Get changed notebook files + id: changed-notebooks + uses: tj-actions/changed-files@v44 + with: + files: '**/*.ipynb' + separator: ' ' - # - name: Run pytest on Python files - # if: steps.changed-py-files.outputs.any_changed == 'true' - # run: | - # pytest Tests/dea_tools + - name: Print changed notebook files + if: steps.changed-notebooks.outputs.any_changed == 'true' + run: | + echo "Changed notebook files:" + echo "${{ steps.changed-notebooks.outputs.all_changed_files }}" + + - name: Run the Notebook tests + if: steps.changed-notebooks.outputs.any_changed == 'true' + run: | + sudo chown -R 1000:100 ./dea-notebooks + cd ./dea-notebooks + docker run --rm \ + --net=host \ + --env DATACUBE_DB_URL \ + --env AWS_SESSION_TOKEN \ + --env AWS_REGION \ + --env AWS_ACCESS_KEY_ID \ + --env AWS_SECRET_ACCESS_KEY \ + --env AWS_SESSION_TOKEN \ + --volume ${GITHUB_WORKSPACE}/dea-notebooks:/home/jovyan/dea-notebooks \ + --volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \ + --env GDAL_HTTP_MAX_RETRY=3 \ + --entrypoint /bin/bash \ + 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable \ + -c " + set -ex + set -o pipefail + cd /home/jovyan/dea-notebooks + pip3 install ./Tools + pytest Tests/dea_tools + pytest --durations=10 --nbval-lax ${{ steps.changed-notebooks.outputs.all_changed_files }} + " \ No newline at end of file diff --git a/.github/workflows/test_notebooks.yml b/.github/workflows/test_notebooks.yml index 045d60f09..fb7c465ac 100644 --- a/.github/workflows/test_notebooks.yml +++ b/.github/workflows/test_notebooks.yml @@ -1,53 +1,53 @@ -name: Test notebooks - -on: - push: - branches: [ develop, stable, nbtests ] - paths-ignore: - - '**/*.md' # ignore markdown files - - '**/*.rst' # ignore restructured text files - - '.github/**' # ignore anything in .github folder - - '!.github/workflows/test_notebooks.yml' # except test_notebooks.yml - pull_request: - branches: [ develop, stable ] - paths-ignore: - - '**/*.md' - - '**/*.rst' - - '.github/**' - - '!.github/workflows/test_notebooks.yml' - -permissions: - id-token: write # This is required for requesting the JSON web token - contents: read # This is required for actions/checkout - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - path: dea-notebooks - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly - aws-region: ap-southeast-2 - - - name: Copy tide modelling files with the AWS CLI - run: | - aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014 - aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide - - - name: Login to Amazon ECR Private - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Set up Datacube and test - run: | - sudo chown -R 1000:100 ./dea-notebooks - cd ./dea-notebooks - CURRENT_UID=1000:100 docker-compose up -d - docker-compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh - docker-compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh +# name: Test notebooks + +# on: +# push: +# branches: [ develop, stable, nbtests ] +# paths-ignore: +# - '**/*.md' # ignore markdown files +# - '**/*.rst' # ignore restructured text files +# - '.github/**' # ignore anything in .github folder +# - '!.github/workflows/test_notebooks.yml' # except test_notebooks.yml +# pull_request: +# branches: [ develop, stable ] +# paths-ignore: +# - '**/*.md' +# - '**/*.rst' +# - '.github/**' +# - '!.github/workflows/test_notebooks.yml' + +# permissions: +# id-token: write # This is required for requesting the JSON web token +# contents: read # This is required for actions/checkout + +# jobs: +# test: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# with: +# path: dea-notebooks + +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v1 +# with: +# role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly +# aws-region: ap-southeast-2 + +# - name: Copy tide modelling files with the AWS CLI +# run: | +# aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014 +# aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide + +# - name: Login to Amazon ECR Private +# id: login-ecr +# uses: aws-actions/amazon-ecr-login@v1 + +# - name: Set up Datacube and test +# run: | +# sudo chown -R 1000:100 ./dea-notebooks +# cd ./dea-notebooks +# CURRENT_UID=1000:100 docker-compose up -d +# docker-compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh +# docker-compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh