Add new or updated feeds from Google Sheets/Form #374
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Add new or updated feeds from Google Sheets/Form | |
| on: | |
| workflow_dispatch: | |
| # schedule: | |
| # - cron: '55 3 * * *' # Run every night | |
| env: | |
| DATE_FORMAT: "[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}" # this is the format we need to compare dates between the CSV and the local system. | |
| DATE_FORMAT_DESIRED: "MM/dd/yyyy" | |
| USERNAME: "github-actions[bot]" # GitHub username that will create the PR | |
| USERNAME_EMAIL: "41898282+github-actions[bot]@users.noreply.github.com" | |
| BASE: "main" | |
| REVIEWERS_JSON: "[\"emmambd\"]" # List of GitHub usernames of the reviewers, in a JSON array : ["username1", "username2"] | |
| jobs: | |
| add-new-updated-feeds: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup global variables | |
| id: global_vars | |
| run: | | |
| echo "TODAYS_DATE=$(date +%m/%d/%Y)" >> $GITHUB_ENV # Ex.: 07/27/2023 | |
| echo "TODAYS_DAY=$(date '+%d')" >> $GITHUB_ENV # Ex.: 27 | |
| echo "TODAYS_MONTH=$(date '+%m')" >> $GITHUB_ENV # Ex.: 07 | |
| echo "TODAYS_YEAR=$(date '+%Y')" >> $GITHUB_ENV # Ex.: 2023 | |
| echo "YESTERDAYS_DATE=$(date -d "yesterday" +"%m/%d/%Y")" >> $GITHUB_ENV # Ex.: 07/26/2023 | |
| echo "RUN_ID=$GITHUB_RUN_ID" >> $GITHUB_ENV | |
| - name: Create branch name | |
| id: create_branch_name | |
| run: | | |
| echo "BRANCH=${{ env.TODAYS_YEAR }}-${{ env.TODAYS_MONTH }}-${{ env.TODAYS_DAY }}-SOURCES" >> $GITHUB_OUTPUT # Branch name | |
| - name: Load secrets from 1Password | |
| id: onepw_secrets | |
| uses: 1password/[email protected] | |
| with: | |
| export-env: true # Export loaded secrets as environment variables | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| CREDENTIALS: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/ifkeehu5gzi7wy5ub5qvwkaire/credential" | |
| # The URL to obtain the csv file with the data of the google form. Taken from the "GiHub Actions — URLs" entry in 1password | |
| CSV_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/qkn5esttmtojawglm4l6t2bqaa/al2gjfhiuddabkp7o26hszuvia" | |
| SLACK_WEBHOOK_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Slack webhook URLs/rdpfgrmnbxqaelgi5oky3lryz4/internal-add-feeds" | |
| - name: Checkout repo | |
| id: checkout_repo | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.BASE }} | |
| fetch-depth: 0 | |
| token: ${{ env.CREDENTIALS }} | |
| - name: Create new branch | |
| shell: bash | |
| run: | | |
| git pull | |
| git checkout -b ${{ steps.create_branch_name.outputs.BRANCH }} | |
| git reset --hard ${{ env.BASE }} | |
| git clean -df | |
| - name: Download CSV and process each lines | |
| id: process-csv | |
| run: | | |
| cd ${{ github.workspace }}/scripts | |
| OUTPUT=$(swift process_csv_in_github_action.swift "${{ env.CSV_URL }}" "${{ env.YESTERDAYS_DATE }}" "${{ env.DATE_FORMAT }}" "${{ env.DATE_FORMAT_DESIRED }}") | |
| echo "PYTHON_SCRIPT_ARGS=${OUTPUT}" >> $GITHUB_OUTPUT | |
| # Count the number of feeds in the output | |
| COUNT=$(echo "$input_string" | grep -o '§' | wc -l) | |
| COUNT=$((COUNT + 1)) | |
| echo "NUMBER_OF_FEEDS=${COUNT}" >> $GITHUB_OUTPUT | |
| - name: Setup Python | |
| if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' # install the python version needed | |
| - name: Create + activate a Python virtual env & run script | |
| if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/tools | |
| PYTHONIOENCODING: "utf8" #ascii | |
| shell: bash | |
| run: | | |
| echo "BEGIN VIRTUAL ENV && SETUP PIP" | |
| echo -e "\n" | |
| python -m venv env | |
| source env/bin/activate | |
| pip install --upgrade pip | |
| pip install virtualenv --quiet | |
| pip install gtfs_kit --quiet | |
| pip install unidecode --quiet | |
| echo "BEGIN" | |
| sections=$(echo '${{ steps.process-csv.outputs.PYTHON_SCRIPT_ARGS }}' | sed 's/""/"/g' | sed "s/’/'/g" | awk -F'§' '{for (i=1; i<=NF; i++) print $i}') | |
| set -x # this should force GitHub to output every echo commands | |
| for section in "${sections[@]}"; do | |
| echo "processing line: ${section}" | |
| output=$(eval "python -c 'from tools.operations import *; ${section}'") | |
| echo "$output" | |
| done | |
| set +x | |
| echo "DONE" | |
| git status | |
| - name: Commit, push, and create PR | |
| id: commit-push-create-pr | |
| if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' | |
| run: | | |
| BRANCH_NAME=${{ steps.create_branch_name.outputs.BRANCH }} | |
| # Fetch the latest changes from main | |
| git fetch origin main | |
| # Create or reset the branch to the current main | |
| git checkout -B "$BRANCH_NAME" origin/main | |
| # Configure git | |
| git config --global user.name "${{ env.USERNAME }}" | |
| git config --global user.email "${{ env.USERNAME_EMAIL }}" | |
| # Add changes | |
| git add . | |
| # Count new files | |
| NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") | |
| # Commit changes | |
| git commit -m "Automated commit — New/Updated feed(s)" | |
| # Force push changes to the branch | |
| git push -f origin "$BRANCH_NAME" | |
| # Check if a PR already exists | |
| EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') | |
| if [ -n "$EXISTING_PR" ]; then | |
| echo "Pull request already exists. Updating PR #$EXISTING_PR" | |
| PR_URL=$(gh pr view $EXISTING_PR --json url -q '.url') | |
| else | |
| echo "Creating new pull request" | |
| # Create pull request and capture the URL | |
| PR_URL=$(gh pr create --title "Automated Pull Request — New/Updated feed(s)" \ | |
| --body "This pull request contains new or updated feed(s)" \ | |
| --base main \ | |
| --head "$BRANCH_NAME") | |
| fi | |
| # Set the PR URL as an output | |
| echo "pr-url=$PR_URL" >> $GITHUB_OUTPUT | |
| echo "new-file-count=$NEW_FILE_COUNT" >> $GITHUB_OUTPUT | |
| env: | |
| GITHUB_TOKEN: ${{ env.CREDENTIALS }} | |
| shell: bash | |
| - name: Post notification in Slack channel of update | |
| if: success() && steps.commit-push.outcome == 'success' | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "channel": "C072DF86VQB", | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "✅ Added/updated ${{ steps.commit-push-create-pr.outputs.new-file-count }} feed(s)", | |
| "emoji": true | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "${{ steps.commit-push-create-pr.outputs.new-file-count }} feed(s) were added or updated in a new branch on the https://github.com/MobilityData/mobility-database-catalogs repo." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "New branch name: `${{ steps.create_branch_name.outputs.BRANCH }}`*Open the corresponding PR:* ${{ steps.commit-push-create-pr.outputs.pr-url }}\n\n*Open the new branch:* https://github.com/MobilityData/mobility-database-catalogs/tree/${{ steps.create_branch_name.outputs.BRANCH }}\n\n" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "See the source document here: https://docs.google.com/spreadsheets/d/1Q96KDppKsn2khdrkraZCQ7T_qRSfwj7WsvqXvuMt4Bc/edit#gid=2061813733" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| - name: Post notification in Slack in case of failure | |
| if: failure() | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "channel": "C072DF86VQB", | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "❌ FAILED — Add/update ${{ steps.commit-push-create-pr.outputs.new-file-count }} feed(s)", | |
| "emoji": true | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "The GitHub action has failed on the https://github.com/MobilityData/mobility-database-catalogs repo." | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "*Open the action run:* https://github.com/MobilityData/mobility-database-catalogs/actions/runs/${{ env.RUN_ID }}" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "See the source document here: https://docs.google.com/spreadsheets/d/1Q96KDppKsn2khdrkraZCQ7T_qRSfwj7WsvqXvuMt4Bc/edit#gid=2061813733" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |