Redeliver failed webhooks to account for temporary errors

See: https://progress.opensuse.org/issues/165144

.github/workflows/redeliver.yaml

@@ -0,0 +1,36 @@
+---
+name: Redeliver failed webhooks
+
+# This workflow runs every 6 hours or when manually triggered.
+on:
+  schedule:
+    - cron: '20 */6 * * *'
+  workflow_dispatch:
+
+# This workflow will use the built in `GITHUB_TOKEN` to check out the repository contents. This grants `GITHUB_TOKEN` permission to do that.
+permissions:
+  contents: read
+
+jobs:
+  redeliver:
+    name: Redeliver failed webhooks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '18.x'
+      - run: npm install octokit
+      - name: Run script
+        env:
+          TOKEN: ${{ secrets.GH_TOKEN_FOR_ACTIONS }}
+          REPO_OWNER: 'os-autoinst'
+          REPO_NAME: 'openQA'
+          HOOK_ID: 'https://build.opensuse.org/trigger/workflow?id=5857'
+          LAST_WEBHOOK_REDELIVERY: 'LAST_WEBHOOK_REDELIVERY'
+
+          WORKFLOW_REPO_NAME: ${{ github.event.repository.name }}
+          WORKFLOW_REPO_OWNER: ${{ github.repository_owner }}
+        run: |
+          node .github/workflows/scripts/redeliver.js
+

.github/workflows/scripts/redeliver.js

@@ -0,0 +1,215 @@
+const { Octokit } = require("octokit");
+
+//
+async function checkAndRedeliverWebhooks() {
+  const TOKEN = process.env.TOKEN;
+  const REPO_OWNER = process.env.REPO_OWNER;
+  const REPO_NAME = process.env.REPO_NAME;
+  const HOOK_ID = process.env.HOOK_ID;
+  const LAST_WEBOOK_REDELIVERY = process.env.LAST_WEBOOK_REDELIVERY;
+
+  const WORKFLOW_REPO_NAME = process.env.WORKFLOW_REPO_NAME;
+  const WORKFLOW_REPO_OWNER = process.env.WORKFLOW_REPO_OWNER;
+
+  // Create an instance of `Octokit` using the token values that were set in the GitHub Actions workflow.
+  const octokit = new Octokit({ 
+    auth: TOKEN,
+  });
+
+  try {
+    const lastStoredRedeliveryTime = await getVariable({
+      variableName: LAST_WEBHOOK_REDELIVERY,
+      repoOwner: WORKFLOW_REPO_OWNER,
+      repoName: WORKFLOW_REPO_NAME,
+      octokit,
+    });
+    // Get the last time this script ran or the current time minus 24 hours.
+    const lastWebhookRedeliveryTime = lastStoredRedeliveryTime || (Date.now() - (24 * 60 * 60 * 1000)).toString();
+    const newWebhookRedeliveryTime = Date.now().toString();
+    const deliveries = await fetchWebhookDeliveriesSince({
+      lastWebhookRedeliveryTime,
+      repoOwner: REPO_OWNER,
+      repoName: REPO_NAME,
+      hookId: HOOK_ID,
+      octokit,
+    });
+
+    // Consolidate deliveries that have the same identifier
+    let deliveriesByGuid = {};
+    for (const delivery of deliveries) {
+      deliveriesByGuid[delivery.guid]
+        ? deliveriesByGuid[delivery.guid].push(delivery)
+        : (deliveriesByGuid[delivery.guid] = [delivery]);
+    }
+    let failedDeliveryIDs = [];
+    for (const guid in deliveriesByGuid) {
+      const deliveries = deliveriesByGuid[guid];
+      const anySucceeded = deliveries.some(
+        (delivery) => delivery.status === "OK"
+      );
+      if (!anySucceeded) {
+        failedDeliveryIDs.push(deliveries[0].id);
+      }
+    }
+
+    // Redeliver any failed deliveries.
+    for (const deliveryId of failedDeliveryIDs) {
+      await redeliverWebhook({
+        deliveryId,
+        repoOwner: REPO_OWNER,
+        repoName: REPO_NAME,
+        hookId: HOOK_ID,
+        octokit,
+      });
+    }
+
+    // Save the last time this was executed
+    await updateVariable({
+      variableName: LAST_WEBHOOK_REDELIVERY,
+      value: newWebhookRedeliveryTime,
+      variableExists: Boolean(lastStoredRedeliveryTime),
+      repoOwner: WORKFLOW_REPO_OWNER,
+      repoName: WORKFLOW_REPO_NAME,
+      octokit,
+    });
+
+    // Log the number of redeliveries.
+    console.log(
+      `Redelivered ${
+        failedDeliveryIDs.length
+      } failed webhook deliveries out of ${
+        deliveries.length
+      } total deliveries since ${Date(lastWebhookRedeliveryTime)}.`
+    );
+  } catch (error) {
+    if (error.response) {
+      console.error(
+        `Failed to check and redeliver webhooks: ${error.response.data.message}`
+      );
+    } else {
+      console.error(error);
+    }
+    // Always throw to ensure the workflow still appears as failed
+    throw(error);
+  }
+}
+
+async function fetchWebhookDeliveriesSince({
+  lastWebhookRedeliveryTime,
+  repoOwner,
+  repoName,
+  hookId,
+  octokit,
+}) {
+  const iterator = octokit.paginate.iterator(
+    "GET /repos/{owner}/{repo}/hooks/{hook_id}/deliveries",
+    {
+      owner: repoOwner,
+      repo: repoName,
+      hook_id: hookId,
+      per_page: 100,
+      headers: {
+        "x-github-api-version": "2022-11-28",
+      },
+    }
+  );
+
+  const deliveries = [];
+
+  for await (const { data } of iterator) {
+    const oldestDeliveryTimestamp = new Date(
+      data[data.length - 1].delivered_at
+    ).getTime();
+
+    if (oldestDeliveryTimestamp < lastWebhookRedeliveryTime) {
+      for (const delivery of data) {
+        if (
+          new Date(delivery.delivered_at).getTime() > lastWebhookRedeliveryTime
+        ) {
+          deliveries.push(delivery);
+        } else {
+          break;
+        }
+      }
+      break;
+    } else {
+      deliveries.push(...data);
+    }
+  }
+
+  return deliveries;
+}
+
+async function redeliverWebhook({
+  deliveryId,
+  repoOwner,
+  repoName,
+  hookId,
+  octokit,
+}) {
+  await octokit.request(
+    "POST /repos/{owner}/{repo}/hooks/{hook_id}/deliveries/{delivery_id}/attempts",
+    {
+      owner: repoOwner,
+      repo: repoName,
+      hook_id: hookId,
+      delivery_id: deliveryId,
+    }
+  );
+}
+
+async function getVariable({ variableName, repoOwner, repoName, octokit }) {
+  try {
+    const {
+      data: { value },
+    } = await octokit.request(
+      "GET /repos/{owner}/{repo}/actions/variables/{name}",
+      {
+        owner: repoOwner,
+        repo: repoName,
+        name: variableName,
+      }
+    );
+    return value;
+  } catch (error) {
+    if (error.status === 404) {
+      return undefined;
+    } else {
+      throw error;
+    }
+  }
+}
+
+async function updateVariable({
+  variableName,
+  value,
+  variableExists,
+  repoOwner,
+  repoName,
+  octokit,
+}) {
+  if (variableExists) {
+    await octokit.request(
+      "PATCH /repos/{owner}/{repo}/actions/variables/{name}",
+      {
+        owner: repoOwner,
+        repo: repoName,
+        name: variableName,
+        value: value,
+      }
+    );
+  } else {
+    await octokit.request("POST /repos/{owner}/{repo}/actions/variables", {
+      owner: repoOwner,
+      repo: repoName,
+      name: variableName,
+      value: value,
+    });
+  }
+}
+
+(async () => {
+  await checkAndRedeliverWebhooks();
+})();
+
+