-
Notifications
You must be signed in to change notification settings - Fork 4
156 lines (155 loc) · 9.43 KB
/
add-collection-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
# This is a test pipeline, that verifies and updates the associations in cmr
name: Add Collection Test
# Controls when the workflow will run
on:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
jobs:
# First job in the workflow installs and verifies the software
build:
name: Test Execution
# The type of runner that the job will run on
runs-on: ubuntu-latest
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Create Output Directory
run: mkdir test_output
- name: Set environment variables
run: |
echo "UAT_OUTPUT_FILE=test_output/output/uat_output_json.txt" >> $GITHUB_ENV
echo "OPS_OUTPUT_FILE=test_output/output/ops_output_json.txt" >> $GITHUB_ENV
echo "OUTPUT_DIR=test_output/output" >> $GITHUB_ENV
echo "UAT_SUCCESS_FILE=uat_success.txt" >> $GITHUB_ENV
echo "OPS_SUCCESS_FILE=ops_success.txt" >> $GITHUB_ENV
echo "UAT_FAILURE_FILE=uat_fail.txt" >> $GITHUB_ENV
echo "OPS_FAILURE_FILE=ops_fail.txt" >> $GITHUB_ENV
echo "SLACK_REPORT_CONTENT=" >> $GITHUB_ENV
- name: Set environment variables for notebook
run: |
echo "OPS_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV
echo "OPS_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV
echo "UAT_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV
echo "UAT_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV
- name: Create prerequisites
run: |
mkdir ${{ env.OUTPUT_DIR }}
realpath ${{ env.OUTPUT_DIR }}
- name: Install dependencies
run: |
pip3 install --upgrade pip
pip3 install black
pip3 install netCDF4
pip3 install git+https://github.com/nasa/harmony-py.git
pip3 install git+https://github.com/podaac/cmr-umm-updater.git
pip3 install git+https://github.com/podaac/cmr-association-diff.git
pip3 install python-dateutil --upgrade
- name: Run CMR Association diff scripts
run: |
cmr_association_diff -e uat -t service -a "cmr/uat_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.UAT_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }}
cmr_association_diff -e ops -t service -a "cmr/ops_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.OPS_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }}
- name: Run Add Collection Test
run: |
python3 add_collection_test.py -e uat -i ${{ env.UAT_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }}
python3 add_collection_test.py -e ops -i ${{ env.OPS_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }}
- name: Check UAT files
id: check_UAT_output_files
run: |
echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
echo
echo
echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
echo
echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
- name: Check OPS files
id: check_OPS_output_files
run: |
echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
echo
echo
echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
echo
echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
- name: Slack Report update - Add info for UAT check - SUCCESS
if: |
steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true'
run: |
echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test UAT result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV
- name: Slack Report update - Add info for UAT check - FAILURE
if: |
steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true'
run: |
echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test UAT result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV
- name: Slack Report update - Add info for OPS check - SUCCESS
if: |
steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true'
run: |
echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test OPS result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV
- name: Slack Report update - Add info for OPS check - FAILURE
if: |
steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true'
run: |
echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\ncollection-test OPS result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV
- name: Slack Report update - Add action link on Failure
if: |
steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true' ||
steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true'
run: |
echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\nExecution link:\ngithub.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV
echo ${{ env.SLACK_REPORT_CONTENT }}
- name: Send Report to Slack
uses: slackapi/[email protected]
if: |
env.SLACK_REPORT_CONTENT != ''
with:
payload: |
{
"status": "${{ env.SLACK_REPORT_CONTENT }}"
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BOT_WEBHOOK }}
- name: Remove UAT collections on FAIL
if: |
steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true'
run: |
python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e uat -i '${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }}
- name: Remove OPS collections on FAIL
if: |
steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true'
run: |
python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e ops -i '${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }}
- name: Update UAT collections on SUCCESS
if: |
steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true'
# Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character
run: |
vi -escwq ./cmr/uat_associations.txt
cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} >> ./cmr/uat_associations.txt
git config user.name "${GITHUB_ACTOR}"
git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
git commit -am "Update uat_associations.txt with new collections"
git push
- name: Update OPS collections on SUCCESS
if: |
steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true'
# Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character
run: |
vi -escwq ./cmr/ops_associations.txt
cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} >> ./cmr/ops_associations.txt
git config user.name "${GITHUB_ACTOR}"
git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
git commit -am "Update ops_associations.txt with new collections"
git push