From cc7c6e496b077797242042f58772d380416cf8d9 Mon Sep 17 00:00:00 2001 From: Liv Dywan Date: Wed, 21 Aug 2024 12:52:32 +0200 Subject: [PATCH] Redeliver failed webhooks to account for temporary errors See: https://progress.opensuse.org/issues/165144 --- .github/workflows/redeliver.yaml | 36 +++++ .github/workflows/scripts/redeliver.js | 215 +++++++++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 .github/workflows/redeliver.yaml create mode 100644 .github/workflows/scripts/redeliver.js diff --git a/.github/workflows/redeliver.yaml b/.github/workflows/redeliver.yaml new file mode 100644 index 000000000000..411dbb771159 --- /dev/null +++ b/.github/workflows/redeliver.yaml @@ -0,0 +1,36 @@ +--- +name: Redeliver failed webhooks + +# This workflow runs every 6 hours or when manually triggered. +on: + schedule: + - cron: '20 */6 * * *' + workflow_dispatch: + +# This workflow will use the built in `GITHUB_TOKEN` to check out the repository contents. This grants `GITHUB_TOKEN` permission to do that. +permissions: + contents: read + +jobs: + redeliver: + name: Redeliver failed webhooks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '18.x' + - run: npm install octokit + - name: Run script + env: + TOKEN: ${{ secrets.GH_TOKEN_FOR_ACTIONS }} + REPO_OWNER: 'os-autoinst' + REPO_NAME: 'openQA' + HOOK_ID: 'https://build.opensuse.org/trigger/workflow?id=5857' + LAST_WEBHOOK_REDELIVERY: 'LAST_WEBHOOK_REDELIVERY' + + WORKFLOW_REPO_NAME: ${{ github.event.repository.name }} + WORKFLOW_REPO_OWNER: ${{ github.repository_owner }} + run: | + node .github/workflows/scripts/redeliver.js + diff --git a/.github/workflows/scripts/redeliver.js b/.github/workflows/scripts/redeliver.js new file mode 100644 index 000000000000..5e70a52ae1ec --- /dev/null +++ b/.github/workflows/scripts/redeliver.js @@ -0,0 +1,215 @@ +const { Octokit } = require("octokit"); + +// +async function checkAndRedeliverWebhooks() { + const TOKEN = process.env.TOKEN; + const REPO_OWNER = process.env.REPO_OWNER; + const REPO_NAME = process.env.REPO_NAME; + const HOOK_ID = process.env.HOOK_ID; + const LAST_WEBOOK_REDELIVERY = process.env.LAST_WEBOOK_REDELIVERY; + + const WORKFLOW_REPO_NAME = process.env.WORKFLOW_REPO_NAME; + const WORKFLOW_REPO_OWNER = process.env.WORKFLOW_REPO_OWNER; + + // Create an instance of `Octokit` using the token values that were set in the GitHub Actions workflow. + const octokit = new Octokit({ + auth: TOKEN, + }); + + try { + const lastStoredRedeliveryTime = await getVariable({ + variableName: LAST_WEBHOOK_REDELIVERY, + repoOwner: WORKFLOW_REPO_OWNER, + repoName: WORKFLOW_REPO_NAME, + octokit, + }); + // Get the last time this script ran or the current time minus 24 hours. + const lastWebhookRedeliveryTime = lastStoredRedeliveryTime || (Date.now() - (24 * 60 * 60 * 1000)).toString(); + const newWebhookRedeliveryTime = Date.now().toString(); + const deliveries = await fetchWebhookDeliveriesSince({ + lastWebhookRedeliveryTime, + repoOwner: REPO_OWNER, + repoName: REPO_NAME, + hookId: HOOK_ID, + octokit, + }); + + // Consolidate deliveries that have the same identifier + let deliveriesByGuid = {}; + for (const delivery of deliveries) { + deliveriesByGuid[delivery.guid] + ? deliveriesByGuid[delivery.guid].push(delivery) + : (deliveriesByGuid[delivery.guid] = [delivery]); + } + let failedDeliveryIDs = []; + for (const guid in deliveriesByGuid) { + const deliveries = deliveriesByGuid[guid]; + const anySucceeded = deliveries.some( + (delivery) => delivery.status === "OK" + ); + if (!anySucceeded) { + failedDeliveryIDs.push(deliveries[0].id); + } + } + + // Redeliver any failed deliveries. + for (const deliveryId of failedDeliveryIDs) { + await redeliverWebhook({ + deliveryId, + repoOwner: REPO_OWNER, + repoName: REPO_NAME, + hookId: HOOK_ID, + octokit, + }); + } + + // Save the last time this was executed + await updateVariable({ + variableName: LAST_WEBHOOK_REDELIVERY, + value: newWebhookRedeliveryTime, + variableExists: Boolean(lastStoredRedeliveryTime), + repoOwner: WORKFLOW_REPO_OWNER, + repoName: WORKFLOW_REPO_NAME, + octokit, + }); + + // Log the number of redeliveries. + console.log( + `Redelivered ${ + failedDeliveryIDs.length + } failed webhook deliveries out of ${ + deliveries.length + } total deliveries since ${Date(lastWebhookRedeliveryTime)}.` + ); + } catch (error) { + if (error.response) { + console.error( + `Failed to check and redeliver webhooks: ${error.response.data.message}` + ); + } else { + console.error(error); + } + // Always throw to ensure the workflow still appears as failed + throw(error); + } +} + +async function fetchWebhookDeliveriesSince({ + lastWebhookRedeliveryTime, + repoOwner, + repoName, + hookId, + octokit, +}) { + const iterator = octokit.paginate.iterator( + "GET /repos/{owner}/{repo}/hooks/{hook_id}/deliveries", + { + owner: repoOwner, + repo: repoName, + hook_id: hookId, + per_page: 100, + headers: { + "x-github-api-version": "2022-11-28", + }, + } + ); + + const deliveries = []; + + for await (const { data } of iterator) { + const oldestDeliveryTimestamp = new Date( + data[data.length - 1].delivered_at + ).getTime(); + + if (oldestDeliveryTimestamp < lastWebhookRedeliveryTime) { + for (const delivery of data) { + if ( + new Date(delivery.delivered_at).getTime() > lastWebhookRedeliveryTime + ) { + deliveries.push(delivery); + } else { + break; + } + } + break; + } else { + deliveries.push(...data); + } + } + + return deliveries; +} + +async function redeliverWebhook({ + deliveryId, + repoOwner, + repoName, + hookId, + octokit, +}) { + await octokit.request( + "POST /repos/{owner}/{repo}/hooks/{hook_id}/deliveries/{delivery_id}/attempts", + { + owner: repoOwner, + repo: repoName, + hook_id: hookId, + delivery_id: deliveryId, + } + ); +} + +async function getVariable({ variableName, repoOwner, repoName, octokit }) { + try { + const { + data: { value }, + } = await octokit.request( + "GET /repos/{owner}/{repo}/actions/variables/{name}", + { + owner: repoOwner, + repo: repoName, + name: variableName, + } + ); + return value; + } catch (error) { + if (error.status === 404) { + return undefined; + } else { + throw error; + } + } +} + +async function updateVariable({ + variableName, + value, + variableExists, + repoOwner, + repoName, + octokit, +}) { + if (variableExists) { + await octokit.request( + "PATCH /repos/{owner}/{repo}/actions/variables/{name}", + { + owner: repoOwner, + repo: repoName, + name: variableName, + value: value, + } + ); + } else { + await octokit.request("POST /repos/{owner}/{repo}/actions/variables", { + owner: repoOwner, + repo: repoName, + name: variableName, + value: value, + }); + } +} + +(async () => { + await checkAndRedeliverWebhooks(); +})(); + +