workflows/check-by-name: Better error when base branch also fails

Previously, even if the check also failed on the base branch, it looked
like the PR introduced the failure.

We can easily have a better error message for such cases.

Meanwhile this also paves the road for something like
#256788

.github/workflows/check-by-name.yml

@@ -21,6 +21,16 @@ jobs:
         with:
           # pull_request_target checks out the base branch by default
           ref: refs/pull/${{ github.event.pull_request.number }}/merge
+          # Fetches the merge commit and its parents
+          fetch-depth: 2
+      - name: Determining PR git hashes
+        run: |
+          echo "mergedSha=$(git rev-parse HEAD)" >> "$GITHUB_ENV"
+
+          # For pull_request_target this is the same as $GITHUB_SHA
+          echo "baseSha=$(git rev-parse HEAD^1)" >> "$GITHUB_ENV"
+
+          # We don't need it here, but the PR head would be HEAD^2
       - uses: cachix/install-nix-action@v23
       - name: Determining channel to use for dependencies
         run: |
@@ -51,4 +61,60 @@ jobs:
           # Passing --max-jobs 0 makes sure that we won't build anything
           nix-build "$nixpkgs" -A tests.nixpkgs-check-by-name --max-jobs 0
       - name: Running nixpkgs-check-by-name
-        run: result/bin/nixpkgs-check-by-name .
+        run: |
+          echo "Checking whether the check succeeds on the base branch $GITHUB_BASE_REF"
+          git checkout -q "$baseSha"
+          if result/bin/nixpkgs-check-by-name .; then
+            baseSuccess=1
+          else
+            baseSuccess=
+          fi
+
+          echo "Checking whether the check succeeds on the merged PR"
+          git checkout -q "$mergedSha"
+          if result/bin/nixpkgs-check-by-name .; then
+            mergedSuccess=1
+          else
+            mergedSuccess=
+          fi
+
+          # Print a markdown summary in GitHub actions
+          resultToEmoji() {
+            if [[ -n "$1" ]]; then
+              echo ":heavy_check_mark:"
+            else
+              echo ":x:"
+            fi
+          }
+          echo "| Tested Nixpkgs | Result |" >> "$GITHUB_STEP_SUMMARY"
+          echo "| --- | --- |" >> "$GITHUB_STEP_SUMMARY"
+          echo "| Base commit ($baseSha) | $(resultToEmoji "$baseSuccess") |" >> "$GITHUB_STEP_SUMMARY"
+          echo "| Merged commit ($mergedSha) | $(resultToEmoji "$mergedSuccess") |" >> "$GITHUB_STEP_SUMMARY"
+
+          # Print a summary
+          resultToText() {
+            if [[ -n "$1" ]]; then
+              echo "[SUCCESS]"
+            else
+              echo "[FAIL]"
+            fi
+          }
+          echo "[$(resultToText "$baseSuccess")] On $GITHUB_BASE_REF ($baseSha)"
+          echo "[$(resultToText "$mergedSuccess")] After merging this PR ($mergedSha)"
+
+          # Print a nice message and fail when appropriate
+          if [[ -n "$baseSuccess" ]]; then
+            if [[ -n "$mergedSuccess" ]]; then
+              echo "The check succeeds both the base branch and after merging this PR"
+            else
+              echo "The check succeeds on the base branch but fails after merging this PR"
+              exit 1
+            fi
+          else
+            if [[ -n "$mergedSuccess" ]]; then
+              echo "The check fails on the base branch, but this PR fixes it, nicely done!"
+            else
+              echo "The check fails both on the base branch and after merging this PR, unknown if only this PRs changes would satisfy the check, the base branch needs to be fixed first."
+              exit 1
+            fi
+          else