-
Notifications
You must be signed in to change notification settings - Fork 129
151 lines (135 loc) · 5.83 KB
/
test_notebooks.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
name: Test DEA Notebooks with RDS Connection
on:
push:
branches: [ develop, stable, nbtests, testing_tests ]
paths-ignore:
- '**/*.md' # ignore markdown files
- '**/*.rst' # ignore restructured text files
- '.github/**' # ignore anything in .github folder
- '!.github/workflows/test_notebooks.yml' # except test_notebooks.yml
pull_request:
branches: [ develop, stable ]
paths-ignore:
- '**/*.md'
- '**/*.rst'
- '.github/**'
- '!.github/workflows/test_notebooks.yml'
permissions:
id-token: write # Required for OIDC authentication to AWS
contents: read
jobs:
test-rds:
runs-on: ubuntu-latest
# Automatically activate our mamba environment when it's ready
defaults:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v4
with:
path: dea-notebooks
# - uses: mamba-org/setup-micromamba@v1
# with:
# micromamba-version: '1.5.1-0'
# environment-name: test-env
# create-args: >-
# python=3.10
# datacube
#
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly
aws-region: ap-southeast-2
- name: Copy tide modelling files with the AWS CLI
run: aws s3 sync s3://dea-non-public-data/tide_models/tide_models tide_models
- name: Login to Amazon ECR Private
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Pull dea-sandbox image from ECR
run: |
docker pull 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox
aws-region: ap-southeast-2
- name: Get Database Credentials
run: |
username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption --query Parameter.Value --output text)
# If we need separate username/password they can be split as follows.
# However exporting them to GitHub Environment variables will leave the Password in the Logs, if further steps aren't taken.
# IFS=: read -r PGUSERNAME PGPASSWORD <<< $username_password
# echo "PGUSERNAME=${PGUSERNAME}" >> $GITHUB_ENV
# echo "PGPASSWORD=${PGPASSWORD}" >> $GITHUB_ENV
echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV
- name: Open Port Forward to RDS
run: |
npx basti connect \
--custom-target-vpc vpc-086904199e505c1f6 \
--custom-target-host db-aurora-dea-sandbox-eks-1.cos5zfpkso9m.ap-southeast-2.rds.amazonaws.com \
--custom-target-port 5432 \
--local-port 5432 &
# Wait until the connection comes up, but, if it doesn't, don't hang forever.
npx wait-on --timeout 120000 --interval 1000 tcp:127.0.0.1:5432
echo "PGPORT=5432" >> $GITHUB_ENV
echo "PGHOST=localhost" >> $GITHUB_ENV
- name: Run the Notebook tests
run: |
sudo chown -R 1000:100 ./dea-notebooks
cd ./dea-notebooks
docker run --rm \
--net=host \
--env DATACUBE_DB_URL \
--env AWS_SESSION_TOKEN \
--env AWS_REGION \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env AWS_SESSION_TOKEN \
--volume ${GITHUB_WORKSPACE}/dea-notebooks:/home/jovyan/dea-notebooks \
--volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \
--env GDAL_HTTP_MAX_RETRY=3 \
--entrypoint /bin/bash \
538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable \
/home/jovyan/dea-notebooks/Tests/test_notebooks.sh
# login-to-amazon-ecr:
# runs-on: ubuntu-latest
# steps:
# - name: Configure AWS credentials
# uses: aws-actions/configure-aws-credentials@v2
# with:
# role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly
# aws-region: ap-southeast-2
# mask-aws-account-id: 'false'
# - name: Login to Amazon ECR
# id: login-ecr
# uses: aws-actions/amazon-ecr-login@v1
# outputs:
# registry: ${{ steps.login-ecr.outputs.registry }}
# docker_username: ${{ steps.login-ecr.outputs.docker_username_538673716275_dkr_ecr_ap_southeast_2_amazonaws_com }} # More information on these outputs can be found below in the 'Docker Credentials' section
# docker_password: ${{ steps.login-ecr.outputs.docker_password_538673716275_dkr_ecr_ap_southeast_2_amazonaws_com }}
#
# run-with-internal-service:
# name: Run tests using dea-sandbox image
# needs: login-to-amazon-ecr
# runs-on: ubuntu-latest
# container:
# image: ${{ needs.login-to-amazon-ecr.outputs.registry }}/geoscienceaustralia/sandbox:stable
# credentials:
# username: ${{ needs.login-to-amazon-ecr.outputs.docker_username }}
# password: ${{ needs.login-to-amazon-ecr.outputs.docker_password }}
# ports:
# - '80:80'
# steps:
# - name: Run steps in container
# run: echo "run steps in container"
# - name: Test RDS Connection
# run: |
# datacube system check
# datacube product list
#
# - name: Test S3 Access
# run: |
# aws s3 ls dea-public-data
# aws s3 ls dea-public-data-dev
# aws s3 cp s3://dea-public-data/index.html .