From 03e1f0e3f8553064377d07729ea8cc9804bc1333 Mon Sep 17 00:00:00 2001
From: Thomas Carmet <8408330+tcarmet@users.noreply.github.com>
Date: Tue, 21 May 2024 22:15:35 +0000
Subject: [PATCH] PTFE-1738 schedule workflow_job events with retries

---
 .../runner-manager/worker/deployment.yaml     |  1 +
 runner_manager/routers/webhook.py             |  4 ++--
 tests/api/test_webhook_router.py              | 23 ++++++++++++++++++-
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/manifests/base/runner-manager/worker/deployment.yaml b/manifests/base/runner-manager/worker/deployment.yaml
index e23319d8..165c2076 100644
--- a/manifests/base/runner-manager/worker/deployment.yaml
+++ b/manifests/base/runner-manager/worker/deployment.yaml
@@ -11,6 +11,7 @@ spec:
             - worker
             - -c
             - runner_manager.jobs.settings
+            - --with-scheduler
           command:
             - rq
           envFrom:
diff --git a/runner_manager/routers/webhook.py b/runner_manager/routers/webhook.py
index 595d11d0..57058d59 100644
--- a/runner_manager/routers/webhook.py
+++ b/runner_manager/routers/webhook.py
@@ -3,7 +3,7 @@
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, Security
 from githubkit.versions.latest.models import WebhookPing
 from githubkit.webhooks import verify
-from rq import Queue
+from rq import Queue, Retry
 
 from runner_manager.dependencies import get_queue, get_settings
 from runner_manager.models.settings import Settings
@@ -62,7 +62,7 @@ def post(
     if event_name in IMPLEMENTED_WEBHOOKS:
         job_name = f"runner_manager.jobs.{event_name}"
 
-        job = queue.enqueue(job_name, webhook)
+        job = queue.enqueue(job_name, webhook, retry=Retry(max=3, interval=[30, 60, 120]))
 
         return WebhookResponse(success=True, message="Job queued", job_id=job.id)
     return WebhookResponse(success=False, message="Not implemented")
diff --git a/tests/api/test_webhook_router.py b/tests/api/test_webhook_router.py
index 5a01bdae..74a515e8 100644
--- a/tests/api/test_webhook_router.py
+++ b/tests/api/test_webhook_router.py
@@ -6,6 +6,7 @@
 from githubkit.webhooks import sign
 from hypothesis import given
 from pytest import fixture
+from rq import Queue
 
 from runner_manager.dependencies import get_settings
 from runner_manager.models.settings import Settings
@@ -29,8 +30,15 @@ def authentified_app(fastapp):
 @given(workflow_job=WorkflowJobCompletedStrategy)
 def test_workflow_job_event(workflow_job, client):
     assert workflow_job.action == "completed"
-    response = client.post("/webhook", content=workflow_job.json(exclude_unset=True))
+    data = workflow_job.json(exclude_unset=True)
+    # Send request without X-GitHub-Event header should return 200
+    # but success should be False
+    response = client.post("/webhook", content=data)
+    assert response.status_code == 200
+    assert response.json()['success'] == False
+    response = client.post("/webhook", content=data, headers={"X-GitHub-Event": "workflow_job"})
     assert response.status_code == 200
+    assert response.json()['success'] == True
 
 
 @given(workflow_job=WorkflowJobCompletedStrategy)
@@ -38,6 +46,18 @@ def test_workflow_job_hypothesis(workflow_job: WorkflowJobCompleted):
     assert workflow_job.action == "completed"
 
 
+@given(workflow_job=WorkflowJobCompletedStrategy)
+def test_webhook_retry_job(workflow_job, client, queue: Queue):
+    assert workflow_job.action == "completed"
+    data = workflow_job.json(exclude_unset=True)
+    response = client.post("/webhook/", content=data, headers={"X-GitHub-Event": "workflow_job"})
+    assert response.status_code == 200
+    job_id = response.json()["job_id"]
+    job = queue.fetch_job(job_id)
+    assert job is not None
+    assert job.retries_left == 3
+
+
 @given(workflow_job=WorkflowJobCompletedStrategy)
 def test_webhook_authentication(workflow_job, client, authentified_app):
     data = workflow_job.json(exclude_unset=True)
@@ -65,6 +85,7 @@ def test_webhook_authentication(workflow_job, client, authentified_app):
     assert response.status_code == 401
 
 
+
 @given(ping=PingStrategy)
 def test_ping_event(ping, client):
     response = client.post("/webhook", content=ping.json(exclude_unset=True))