Scan For New Associations #225
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow scans CMR every 5 minutes to check for new collection associations to the l2ss-py UMM-S record | |
| # in UAT and OPS. If a new association is found, a PR is opened to add the new collection concept id to the | |
| # cmr/*_associations.txt file. | |
| name: Scan For New Associations | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 1 * * *' | |
| jobs: | |
| find_new: | |
| name: Find new associations | |
| runs-on: ubuntu-latest | |
| outputs: | |
| new_uat_associations: ${{ steps.diff.outputs.new_uat_associations }} | |
| new_ops_associations: ${{ steps.diff.outputs.new_ops_associations }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Poetry | |
| uses: abatilo/actions-poetry@v2 | |
| with: | |
| poetry-version: 1.5.1 | |
| - name: Run diff | |
| id: diff | |
| env: | |
| UAT_TOKEN_TEMP: ${{ secrets.LAUNCHPAD_TOKEN_UAT }} | |
| OPS_TOKEN_TEMP: ${{ secrets.LAUNCHPAD_TOKEN_OPS }} | |
| run: | | |
| poetry install | |
| ls $GITHUB_WORKSPACE/tests/cmr/uat/ > $GITHUB_WORKSPACE/tests/cmr/uat_associations.txt | |
| ls $GITHUB_WORKSPACE/tests/cmr/ops/ > $GITHUB_WORKSPACE/tests/cmr/ops_associations.txt | |
| cat $GITHUB_WORKSPACE/tests/cmr/uat_associations.txt | |
| cat $GITHUB_WORKSPACE/tests/cmr/ops_associations.txt | |
| echo "new_uat_associations=$(poetry run cmr_association_diff -e uat -t service -p POCLOUD -n 'PODAAC L2 Cloud Subsetter' -a $GITHUB_WORKSPACE/tests/cmr/uat_associations.txt --token $UAT_TOKEN_TEMP)" >> $GITHUB_OUTPUT | |
| echo "new_ops_associations=$(poetry run cmr_association_diff -e ops -t service -p POCLOUD -n 'PODAAC L2 Cloud Subsetter' -a $GITHUB_WORKSPACE/tests/cmr/ops_associations.txt --token $OPS_TOKEN_TEMP)" >> $GITHUB_OUTPUT | |
| rm $GITHUB_WORKSPACE/tests/cmr/uat_associations.txt | |
| rm $GITHUB_WORKSPACE/tests/cmr/ops_associations.txt | |
| #echo "new_uat_associations=[\"C1238658088-POCLOUD\",\"C1233154410-GES_DISC\",\"C1234666458-GES_DISC\",\"C1238621111-POCLOUD\",\"C1234666454-GES_DISC\",\"C1254989897-LARC_ASDC\",\"C1234666476-GES_DISC\",\"C1236469823-GES_DISC\",\"C1238621219-POCLOUD\",\"C1243175554-POCLOUD\"]" >> $GITHUB_OUTPUT | |
| #echo "new_uat_associations=[\"C1238658088-POCLOUD\",\"C1233154410-GES_DISC\"]" >> $GITHUB_OUTPUT | |
| #echo "new_ops_associations=[\"C1968979997-POCLOUD\"]" >> $GITHUB_OUTPUT | |
| open_pr_uat: | |
| needs: find_new | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| concept_id: ${{ fromJson(needs.find_new.outputs.new_uat_associations) }} | |
| runs-on: ubuntu-latest | |
| name: UAT ${{ matrix.concept_id }} | |
| steps: | |
| - name: Get collection ShortName for readability | |
| run: | | |
| cd $GITHUB_WORKSPACE | |
| MAX_RETRIES=10 | |
| BACKOFF_DELAY=5 | |
| RETRY_COUNT=0 | |
| while [ "$RETRY_COUNT" -lt "$MAX_RETRIES" ]; do | |
| curl "https://cmr.uat.earthdata.nasa.gov/search/collections.umm_json?concept_id=${{ matrix.concept_id }}&token=${{ secrets.LAUNCHPAD_TOKEN_UAT }}" >cmr_metadata.json | |
| SHORT_NAME=$(jq '.items[0].umm.ShortName' cmr_metadata.json) | |
| SHORT_NAME=$(echo "$SHORT_NAME" | sed 's/"//g') | |
| if [ -n "$SHORT_NAME" ] && [ "$SHORT_NAME" != "null" ]; then | |
| echo "Short Name: $SHORT_NAME" | |
| break | |
| else | |
| ((RETRY_COUNT++)) | |
| echo "Retry #$RETRY_COUNT: SHORT_NAME is null. Retrying in $BACKOFF_DELAY seconds..." | |
| sleep "$BACKOFF_DELAY" | |
| BACKOFF_DELAY=$((BACKOFF_DELAY * 2)) | |
| fi | |
| done | |
| #curl "https://cmr.uat.earthdata.nasa.gov/search/collections.umm_json?concept_id=${{ matrix.concept_id }}&token=${{ secrets.LAUNCHPAD_TOKEN_UAT }}" >cmr_metadata.json | |
| #SHORT_NAME=$(jq '.items[0].umm.ShortName' cmr_metadata.json) | |
| #SHORT_NAME=$(echo "$SHORT_NAME" | sed 's/"//g') | |
| echo "SHORT_NAME=$SHORT_NAME">>$GITHUB_ENV | |
| - uses: getsentry/action-github-app-token@v2 | |
| name: Get GH App token | |
| id: podaac-cicd | |
| with: | |
| app_id: ${{ secrets.CICD_APP_ID }} | |
| private_key: ${{ secrets.CICD_APP_PRIVATE_KEY }} | |
| - uses: actions/checkout@v3 | |
| - name: Update uat_associations | |
| run: | | |
| mkdir -p $GITHUB_WORKSPACE/tests/cmr/uat | |
| echo "${{ matrix.concept_id }}" > $GITHUB_WORKSPACE/tests/cmr/uat/${{ matrix.concept_id }} | |
| - name: Create Pull Request for ${{ env.SHORT_NAME }} | |
| id: cpr | |
| uses: peter-evans/create-pull-request@v5 | |
| with: | |
| token: ${{ steps.podaac-cicd.outputs.token }} | |
| commit-message: Add ${{ matrix.concept_id }} | |
| committer: GitHub <[email protected]> | |
| author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> | |
| branch: diff/uat/${{ matrix.concept_id }} | |
| delete-branch: true | |
| title: UAT ${{ matrix.concept_id }} (${{ env.SHORT_NAME }}) | |
| body: | | |
| New association between l2ss-py and ${{ matrix.concept_id }} found in UAT. | |
| Beginning verification of collection. | |
| labels: | | |
| unverified | |
| autotest | |
| - name: Enable Pull Request Automerge | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: gh pr merge --merge --auto ${{ steps.cpr.outputs.pull-request-number }} | |
| open_pr_ops: | |
| needs: find_new | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| concept_id: ${{ fromJson(needs.find_new.outputs.new_ops_associations) }} | |
| runs-on: ubuntu-latest | |
| name: OPS ${{ matrix.concept_id }} | |
| steps: | |
| - name: Get collection ShortName for readability | |
| run: | | |
| MAX_RETRIES=10 | |
| BACKOFF_DELAY=5 | |
| RETRY_COUNT=0 | |
| cd $GITHUB_WORKSPACE | |
| while [ "$RETRY_COUNT" -lt "$MAX_RETRIES" ]; do | |
| curl "https://cmr.earthdata.nasa.gov/search/collections.umm_json?concept_id=${{ matrix.concept_id }}&token=${{ secrets.LAUNCHPAD_TOKEN_OPS }}" >cmr_metadata.json | |
| SHORT_NAME=$(jq '.items[0].umm.ShortName' cmr_metadata.json) | |
| SHORT_NAME=$(echo "$SHORT_NAME" | sed 's/"//g') | |
| if [ -n "$SHORT_NAME" ] && [ "$SHORT_NAME" != "null" ]; then | |
| echo "Short Name: $SHORT_NAME" | |
| break | |
| else | |
| ((RETRY_COUNT++)) | |
| echo "Retry #$RETRY_COUNT: SHORT_NAME is null. Retrying in $BACKOFF_DELAY seconds..." | |
| sleep "$BACKOFF_DELAY" | |
| BACKOFF_DELAY=$((BACKOFF_DELAY * 2)) | |
| fi | |
| done | |
| #curl "https://cmr.earthdata.nasa.gov/search/collections.umm_json?concept_id=${{ matrix.concept_id }}&token=${{ secrets.LAUNCHPAD_TOKEN_OPS }}" >cmr_metadata.json | |
| #SHORT_NAME=$(jq '.items[0].umm.ShortName' cmr_metadata.json) | |
| #SHORT_NAME=$(echo "$SHORT_NAME" | sed 's/"//g') | |
| echo "SHORT_NAME=$SHORT_NAME">>$GITHUB_ENV | |
| - uses: getsentry/action-github-app-token@v2 | |
| name: Get GH App token | |
| id: podaac-cicd | |
| with: | |
| app_id: ${{ secrets.CICD_APP_ID }} | |
| private_key: ${{ secrets.CICD_APP_PRIVATE_KEY }} | |
| - uses: actions/checkout@v3 | |
| - name: Update ops_associations | |
| run: | | |
| mkdir -p $GITHUB_WORKSPACE/tests/cmr/ops | |
| echo "${{ matrix.concept_id }}" > $GITHUB_WORKSPACE/tests/cmr/ops/${{ matrix.concept_id }} | |
| - name: Create Pull Request | |
| id: cpr | |
| uses: peter-evans/create-pull-request@v5 | |
| with: | |
| token: ${{ steps.podaac-cicd.outputs.token }} | |
| commit-message: Add ${{ matrix.concept_id }} | |
| committer: GitHub <[email protected]> | |
| author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> | |
| branch: diff/ops/${{ matrix.concept_id }} | |
| delete-branch: true | |
| title: OPS ${{ matrix.concept_id }} (${{ env.SHORT_NAME }}) | |
| body: | | |
| New association between l2ss-py and ${{ matrix.concept_id }} found in OPS. | |
| Beginning verification of collection. | |
| labels: | | |
| unverified | |
| autotest | |
| - name: Enable Pull Request Automerge | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: gh pr merge --merge --auto ${{ steps.cpr.outputs.pull-request-number }} |