PTFE-790 Ensure runner gets deleted from github (#408)

.trunk/.gitignore

@@ -2,7 +2,7 @@
 *logs
 *actions
 *notifications
+*tools
 plugins
 user_trunk.yaml
 user.yaml
-tools

.trunk/trunk.yaml

@@ -1,10 +1,10 @@
 version: 0.1
 cli:
-  version: 1.13.0
+  version: 1.14.2
 plugins:
   sources:
     - id: trunk
-      ref: v1.0.0
+      ref: v1.2.1
       uri: https://github.com/trunk-io/plugins
 lint:
   definitions:
@@ -20,18 +20,19 @@ lint:
       paths:
         - tests
   enabled:
+    - [email protected]
     - [email protected]
     - [email protected]
-    - black@22.3.0
-    - isort@5.9.3
-    - [email protected].250
-    - yamllint@1.26.3
-    - [email protected].304
+    - black@23.7.0
+    - isort@5.12.0
+    - [email protected].287
+    - yamllint@1.32.0
+    - [email protected].325
     - [email protected]
     - [email protected]
     - git-diff-check
-    - markdownlint@0.35.0
-    - [email protected].0
+    - markdownlint@0.36.0
+    - [email protected].3
     - [email protected]
   disabled:
     - trufflehog
@@ -42,6 +43,7 @@ lint:
     - terrascan
 runtimes:
   enabled:
+    - [email protected]
     - [email protected]
     - python@>=3.11.4
 actions:

runner_manager/jobs/workflow_job.py

@@ -20,11 +20,11 @@ def completed(webhook: WorkflowJobCompleted) -> int:
     if not runner_group or not runner:
         log.warning(f"Runner {webhook.workflow_job.runner_name} not found")
         return 0
+    github: GitHub = get_github()
     log.info(f"Deleting runner {runner.name} in group {runner_group.name}")
-    delete = runner_group.delete_runner(runner)
+    delete = runner_group.delete_runner(runner, github)
     if runner_group.need_new_runner:
         log.info(f"Runner group {runner_group.name} needs a new runner")
-        github: GitHub = get_github()
         runner_group.create_runner(github)
     return delete
 

runner_manager/models/runner_group.py

@@ -52,7 +52,6 @@ class BaseRunnerGroup(PydanticBaseModel):
 
 
 class RunnerGroup(BaseModel, BaseRunnerGroup):
-
     id: Optional[int] = Field(index=True, default=None)
     name: str = Field(index=True, full_text_search=True, max_length=39)
     organization: str = Field(index=True, full_text_search=True)
@@ -172,12 +171,24 @@ def update_runner(self: Self, webhook: WorkflowJobInProgress) -> Runner:
         runner.save()
         return self.backend.update(runner)
 
-    def delete_runner(self, runner: Runner) -> int:
+    def delete_runner(self, runner: Runner, github: GitHub) -> int:
         """Delete a runner instance.
-
+        Start by checking if the runner still exists on GitHub,
+        delete it if it does, then proceed to delete it from the backend.
         Returns:
             Runner: Runner instance.
         """
+        if runner.id is not None:
+            try:
+                github.rest.actions.get_self_hosted_runner_for_org(
+                    org=self.organization, runner_id=runner.id
+                )
+            except RequestFailed:
+                log.info("Runner does not exist")
+            else:
+                github.rest.actions.delete_self_hosted_runner_from_org(
+                    org=self.organization, runner_id=runner.id
+                )
         return self.backend.delete(runner)
 
     @property
@@ -270,9 +281,9 @@ def healthcheck(
         for runner in runners:
             runner.update_from_github(github)
             if runner.time_to_live_expired(time_to_live):
-                self.delete_runner(runner)
+                self.delete_runner(runner, github)
             if runner.time_to_start_expired(timeout_runner):
-                self.delete_runner(runner)
+                self.delete_runner(runner, github)
         while self.need_new_runner:
             runner: Runner = self.create_runner(github)
             if runner:
@@ -281,7 +292,7 @@ def healthcheck(
         # check if there's more idle runners than the minimum
         while len(idle_runners) > self.min:
             runner = idle_runners.pop()
-            self.delete_runner(runner)
+            self.delete_runner(runner, github)
             log.info(f"Runner {runner.name} deleted")
 
     def reset(self, github: GitHub):
@@ -290,7 +301,7 @@ def reset(self, github: GitHub):
             if runner.id is not None:
                 runner.update_from_github(github)
             if not runner.is_active:
-                self.delete_runner(runner)
+                self.delete_runner(runner, github)
                 self.create_runner(github)
                 self.save()
 
@@ -353,9 +364,9 @@ def delete(
         """
         group: RunnerGroup = cls.get(pk)
         runners: List[Runner] = group.get_runners()
-        for runner in runners:
-            group.delete_runner(runner)
         if github:
+            for runner in runners:
+                group.delete_runner(runner, github)
             group.delete_github_group(github)
         db = cls._get_db(pipeline)