Fix files not being committed (#484)

* Added logging to the standard output

* Removed PR step for testing

* Converted two steps into one, commit+PR

* Removed branch checkout, already done in prev step

* Added output for newly created PR to use in Slack notification

... and brought back the Slack notif steps.

* Added git pull before comitting

* Minor fix for git pull

* Few fix for the PR step

* fix for pr create command

* Added fix for existing branch on remote

* Another fix attempt on the PR step

* Added file count, uncomment Slack notif

.github/workflows/add_new_or_updated_feeds.yml

@@ -88,6 +88,8 @@ jobs:
           PYTHONIOENCODING: "utf8" #ascii
         shell: bash
         run: |
+            echo "BEGIN VIRTUAL ENV && SETUP PIP"
+            echo -e "\n"
             python -m venv env
             source env/bin/activate
             pip install --upgrade pip
@@ -100,33 +102,67 @@ jobs:
             sections=$(echo '${{ steps.process-csv.outputs.PYTHON_SCRIPT_ARGS }}' | sed 's/""/"/g' | sed "s/’/'/g" | awk -F'§' '{for (i=1; i<=NF; i++) print $i}')
             for section in "${sections[@]}"; do
                 echo "processing line: ${section}"
-                eval "python -c 'from tools.operations import *; ${section}'"
-                echo " "
+                echo -e "\n"
+                output=$(eval "python -c 'from tools.operations import *; ${section}'")
+                echo -e "\n"
+                echo "$output"
+                echo -e "\n"
             done
 
-      - name: Commit & push
-        id: commit-push
+            echo "DONE"
+            echo -e "\n"
+            git status
+            echo -e "\n"
+
+      - name: Commit, push, and create PR
+        id: commit-push-create-pr
         if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != ''
-        uses: EndBug/[email protected]
-        with:
-            github_token: ${{ env.CREDENTIALS }}
-            new_branch: ${{ steps.create_branch_name.outputs.BRANCH }}
-            author_name: ${{ env.USERNAME }}
-            author_email: ${{ env.USERNAME_EMAIL }}
-            committer_name: ${{ env.USERNAME }}
-            committer_email: ${{ env.USERNAME_EMAIL }}
-            message: "Automated commit — New/Updated feed(s)"
-
-      - name: Create Pull Request
-        id: create-pr
-        if: success() && steps.commit-push.outcome == 'success'
-        uses: peter-evans/[email protected]
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          branch: ${{ steps.create_branch_name.outputs.BRANCH }}
-          title: "Automated Pull Request — New/Updated feed(s)"
-          body: "This pull request contains new or updated feed(s)"
-          base: main
+        run: |
+          BRANCH_NAME=${{ steps.create_branch_name.outputs.BRANCH }}
+
+          # Fetch the latest changes from main
+          git fetch origin main
+
+          # Create or reset the branch to the current main
+          git checkout -B "$BRANCH_NAME" origin/main
+
+          # Configure git
+          git config --global user.name "${{ env.USERNAME }}"
+          git config --global user.email "${{ env.USERNAME_EMAIL }}"
+          
+          # Add changes
+          git add .
+
+          # Count new files
+          NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A")
+          
+          # Commit changes
+          git commit -m "Automated commit — New/Updated feed(s)"
+          
+          # Force push changes to the branch
+          git push -f origin "$BRANCH_NAME"
+          
+          # Check if a PR already exists
+          EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number')
+          
+          if [ -n "$EXISTING_PR" ]; then
+            echo "Pull request already exists. Updating PR #$EXISTING_PR"
+            PR_URL=$(gh pr view $EXISTING_PR --json url -q '.url')
+          else
+            echo "Creating new pull request"
+            # Create pull request and capture the URL
+            PR_URL=$(gh pr create --title "Automated Pull Request — New/Updated feed(s)" \
+                                  --body "This pull request contains new or updated feed(s)" \
+                                  --base main \
+                                  --head "$BRANCH_NAME")
+          fi
+          
+          # Set the PR URL as an output
+          echo "pr-url=$PR_URL" >> $GITHUB_OUTPUT
+          echo "new-file-count=$NEW_FILE_COUNT" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_TOKEN: ${{ env.CREDENTIALS }}
+        shell: bash
 
       - name: Post notification in Slack channel of update
         if: success() && steps.commit-push.outcome == 'success'
@@ -140,22 +176,22 @@ jobs:
                   "type": "header",
                   "text": {
                     "type": "plain_text",
-                    "text": "✅ Added/updated feed(s)",
+                    "text": "✅ Added/updated ${{ steps.commit-push-create-pr.outputs.new-file-count }} feed(s)",
                     "emoji": true
                   }
                 },
                 {
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "${{ steps.process-csv.outputs.NUMBER_OF_FEEDS }} feed(s) were added or updated in a new branch on the https://github.com/MobilityData/mobility-database-catalogs repo."
+                    "text": "${{ steps.commit-push-create-pr.outputs.new-file-count }} feed(s) were added or updated in a new branch on the https://github.com/MobilityData/mobility-database-catalogs repo."
                   }
                 },
                 {
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "New branch name: `${{ steps.create_branch_name.outputs.BRANCH }}`\n\n*Open the new branch:* https://github.com/MobilityData/mobility-database-catalogs/tree/${{ steps.create_branch_name.outputs.BRANCH }}\n\n*Open the corresponding PR:* ${{ steps.create-pr.outputs.pull-request-url }}"
+                    "text": "New branch name: `${{ steps.create_branch_name.outputs.BRANCH }}`*Open the corresponding PR:* ${{ steps.commit-push-create-pr.outputs.pr-url }}\n\n*Open the new branch:* https://github.com/MobilityData/mobility-database-catalogs/tree/${{ steps.create_branch_name.outputs.BRANCH }}\n\n"
                   }
                 },
                 {
@@ -183,7 +219,7 @@ jobs:
                   "type": "header",
                   "text": {
                     "type": "plain_text",
-                    "text": "❌ FAILED — Add/update feed(s)",
+                    "text": "❌ FAILED — Add/update ${{ steps.commit-push-create-pr.outputs.new-file-count }} feed(s)",
                     "emoji": true
                   }
                 },