PTFE-1738 schedule workflow_job events with retries

manifests/base/runner-manager/worker/deployment.yaml

@@ -11,6 +11,7 @@ spec:
             - worker
             - -c
             - runner_manager.jobs.settings
+            - --with-scheduler
           command:
             - rq
           envFrom:

runner_manager/routers/webhook.py

@@ -3,7 +3,7 @@
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, Security
 from githubkit.versions.latest.models import WebhookPing
 from githubkit.webhooks import verify
-from rq import Queue
+from rq import Queue, Retry
 
 from runner_manager.dependencies import get_queue, get_settings
 from runner_manager.models.settings import Settings
@@ -62,7 +62,9 @@ def post(
     if event_name in IMPLEMENTED_WEBHOOKS:
         job_name = f"runner_manager.jobs.{event_name}"
 
-        job = queue.enqueue(job_name, webhook)
+        job = queue.enqueue(
+            job_name, webhook, retry=Retry(max=3, interval=[30, 60, 120])
+        )
 
         return WebhookResponse(success=True, message="Job queued", job_id=job.id)
     return WebhookResponse(success=False, message="Not implemented")

tests/api/test_webhook_router.py

@@ -6,6 +6,7 @@
 from githubkit.webhooks import sign
 from hypothesis import given
 from pytest import fixture
+from rq import Queue
 
 from runner_manager.dependencies import get_settings
 from runner_manager.models.settings import Settings
@@ -29,15 +30,38 @@ def authentified_app(fastapp):
 @given(workflow_job=WorkflowJobCompletedStrategy)
 def test_workflow_job_event(workflow_job, client):
     assert workflow_job.action == "completed"
-    response = client.post("/webhook", content=workflow_job.json(exclude_unset=True))
+    data = workflow_job.json(exclude_unset=True)
+    # Send request without X-GitHub-Event header should return 200
+    # but success should be False
+    response = client.post("/webhook", content=data)
+    assert response.status_code == 200
+    assert response.json()["success"] is False
+    response = client.post(
+        "/webhook", content=data, headers={"X-GitHub-Event": "workflow_job"}
+    )
     assert response.status_code == 200
+    assert response.json()["success"] is True
 
 
 @given(workflow_job=WorkflowJobCompletedStrategy)
 def test_workflow_job_hypothesis(workflow_job: WorkflowJobCompleted):
     assert workflow_job.action == "completed"
 
 
+@given(workflow_job=WorkflowJobCompletedStrategy)
+def test_webhook_retry_job(workflow_job, client, queue: Queue):
+    assert workflow_job.action == "completed"
+    data = workflow_job.json(exclude_unset=True)
+    response = client.post(
+        "/webhook/", content=data, headers={"X-GitHub-Event": "workflow_job"}
+    )
+    assert response.status_code == 200
+    job_id = response.json()["job_id"]
+    job = queue.fetch_job(job_id)
+    assert job is not None
+    assert job.retries_left == 3
+
+
 @given(workflow_job=WorkflowJobCompletedStrategy)
 def test_webhook_authentication(workflow_job, client, authentified_app):
     data = workflow_job.json(exclude_unset=True)