diff --git a/poetry.lock b/poetry.lock index cb80f253..a0670f17 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1014,31 +1014,29 @@ dev = ["flake8", "markdown", "twine", "wheel"] [[package]] name = "githubkit" -version = "0.10.6" +version = "0.11.2" description = "GitHub SDK for Python" optional = false -python-versions = "^3.8" -files = [] -develop = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "githubkit-0.11.2-py3-none-any.whl", hash = "sha256:d6776c667e37e0a120c003fa0a4c4c9a03c99a87a8a1767d3b67a87e09567933"}, + {file = "githubkit-0.11.2.tar.gz", hash = "sha256:2192cd30f32424d5ac8b104f3fc601c69ff1d4f63e2934eff3873953afca961b"}, +] [package.dependencies] -httpx = ">=0.23.0, <1.0.0" -pydantic = "^1.9.1" -typing-extensions = "^4.3.0" +hishel = ">=0.0.21,<=0.0.24" +httpx = ">=0.23.0,<1.0.0" +pydantic = ">=1.9.1,<2.5.0 || >2.5.0,<2.5.1 || >2.5.1,<3.0.0" +PyJWT = {version = ">=2.4.0,<3.0.0", extras = ["crypto"], optional = true, markers = "extra == \"jwt\" or extra == \"auth-app\" or extra == \"auth\" or extra == \"all\""} +typing-extensions = ">=4.3.0,<5.0.0" [package.extras] -all = ["PyJWT[crypto] (>=2.4.0,<3.0.0)", "anyio (>=3.6.1,<4.0.0)"] -auth = ["PyJWT[crypto] (>=2.4.0,<3.0.0)", "anyio (>=3.6.1,<4.0.0)"] +all = ["PyJWT[crypto] (>=2.4.0,<3.0.0)", "anyio (>=3.6.1,<5.0.0)"] +auth = ["PyJWT[crypto] (>=2.4.0,<3.0.0)", "anyio (>=3.6.1,<5.0.0)"] auth-app = ["PyJWT[crypto] (>=2.4.0,<3.0.0)"] -auth-oauth-device = ["anyio (>=3.6.1,<4.0.0)"] +auth-oauth-device = ["anyio (>=3.6.1,<5.0.0)"] jwt = ["PyJWT[crypto] (>=2.4.0,<3.0.0)"] -[package.source] -type = "git" -url = "https://github.com/yanyongyu/githubkit" -reference = "a4275ac3d3babd64061f3693353db740e6a8e892" -resolved_reference = "a4275ac3d3babd64061f3693353db740e6a8e892" - [[package]] name = "google-api-core" version = "2.12.0" @@ -1311,6 +1309,27 @@ files = [ {file = "hiredis-2.2.3.tar.gz", hash = "sha256:e75163773a309e56a9b58165cf5a50e0f84b755f6ff863b2c01a38918fe92daa"}, ] +[[package]] +name = "hishel" +version = "0.0.24" +description = "Persistent cache implementation for httpx and httpcore" +optional = false +python-versions = ">=3.8" +files = [ + {file = "hishel-0.0.24-py3-none-any.whl", hash = "sha256:8b6e43481485e1938d78bd35c0bcb38646fe8f2e090fedb64b4dc1d6015ffe49"}, + {file = "hishel-0.0.24.tar.gz", hash = "sha256:4ac494c6bfedc431e480ab85d3435d4710230b2ad6092766b6ccf82b1d7e4152"}, +] + +[package.dependencies] +httpx = ">=0.22.0" +typing-extensions = ">=4.8.0" + +[package.extras] +redis = ["redis (==5.0.1)"] +s3 = ["boto3 (>=1.15.0,<=1.15.3)", "boto3 (>=1.15.3)"] +sqlite = ["anysqlite (>=0.0.5)"] +yaml = ["pyyaml (==6.0.1)"] + [[package]] name = "httpcore" version = "0.18.0" @@ -1962,6 +1981,26 @@ files = [ [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pymdown-extensions" version = "10.3.1" @@ -3017,4 +3056,4 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "f411f5f2417a59f034978b28038e0718929a1db757f8ce661a8d8f2548c0adab" +content-hash = "cbea552c7f49988f36a42c16a1e92043c05799244b201e93a9722b0a57eb3c0f" diff --git a/pyproject.toml b/pyproject.toml index 0ea31084..573f4082 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,7 @@ google-cloud-compute = "^1.17.0" boto3 = "^1.28.85" botocore = "^1.31.85" boto3-stubs = { extras = ["ec2"], version = "^1.34.54" } -githubkit = { extras = [ - "auth-app", -], git = "https://github.com/yanyongyu/githubkit", rev = "a4275ac3d3babd64061f3693353db740e6a8e892" } +githubkit = { extras = ["auth-app"], version = "^0.11.2" } rq-scheduler = "^0.13.1" diff --git a/runner_manager/backend/aws.py b/runner_manager/backend/aws.py index 969ea97c..b4047850 100644 --- a/runner_manager/backend/aws.py +++ b/runner_manager/backend/aws.py @@ -2,7 +2,7 @@ from boto3 import client from botocore.exceptions import ClientError -from githubkit.webhooks.types import WorkflowJobEvent +from githubkit.versions.latest.webhooks import WorkflowJobEvent from mypy_boto3_ec2 import EC2Client from pydantic import Field from redis_om import NotFoundError diff --git a/runner_manager/backend/base.py b/runner_manager/backend/base.py index 48a6449a..7f3b63b2 100644 --- a/runner_manager/backend/base.py +++ b/runner_manager/backend/base.py @@ -1,6 +1,6 @@ from typing import List, Literal, Optional -from githubkit.webhooks.types import WorkflowJobEvent +from githubkit.versions.latest.webhooks import WorkflowJobEvent from pydantic import BaseModel, Field from redis_om import NotFoundError diff --git a/runner_manager/backend/docker.py b/runner_manager/backend/docker.py index 73177269..308a81bf 100644 --- a/runner_manager/backend/docker.py +++ b/runner_manager/backend/docker.py @@ -6,7 +6,7 @@ from docker import DockerClient from docker.errors import APIError, NotFound from docker.models.containers import Container -from githubkit.webhooks.types import WorkflowJobEvent +from githubkit.versions.latest.webhooks import WorkflowJobEvent from pydantic import Field from redis_om import NotFoundError diff --git a/runner_manager/backend/gcloud.py b/runner_manager/backend/gcloud.py index 3486dfff..a5973f7e 100644 --- a/runner_manager/backend/gcloud.py +++ b/runner_manager/backend/gcloud.py @@ -4,7 +4,7 @@ import time from typing import List, Literal, MutableMapping, Optional -from githubkit.webhooks.types import WorkflowJobEvent +from githubkit.versions.latest.webhooks import WorkflowJobEvent from google.api_core.exceptions import BadRequest, NotFound from google.api_core.extended_operation import ExtendedOperation from google.cloud.compute import ( diff --git a/runner_manager/clients/github.py b/runner_manager/clients/github.py index 4adaef44..898d18f7 100644 --- a/runner_manager/clients/github.py +++ b/runner_manager/clients/github.py @@ -10,19 +10,18 @@ """ - -from datetime import datetime from functools import cached_property -from typing import Dict, List, Literal, Optional +from typing import Dict, List, Optional from githubkit import GitHub as GitHubKit +from githubkit.compat import GitHubModel as GitHubRestModel from githubkit.response import Response -from githubkit.rest import RestNamespace as RestNamespaceKit -from githubkit.rest.actions import ActionsClient as ActionsClientKit -from githubkit.rest.models import GitHubRestModel -from githubkit.utils import UNSET, Missing, exclude_unset -from githubkit.webhooks.models import GitHubWebhookModel -from pydantic import Field +from githubkit.typing import Missing +from githubkit.utils import UNSET, exclude_unset +from githubkit.versions.v2022_11_28.rest import RestNamespace as RestNamespaceKit +from githubkit.versions.v2022_11_28.rest.actions import ( + ActionsClient as ActionsClientKit, +) class RunnerGroup(GitHubRestModel): @@ -43,18 +42,6 @@ class OrgsOrgActionsRunnerGroupsGetResponse200(GitHubRestModel): runner_groups: List[RunnerGroup] -# Missing methods from githubkit.webhooks.models -class WorkflowStepQueued(GitHubWebhookModel): - """Workflow Step (Queued)""" - - name: str = Field(default=...) - status: Literal["queued"] = Field(default=...) - conclusion: None = Field(default=...) - number: int = Field(default=...) - started_at: datetime = Field(default=...) - completed_at: None = Field(default=...) - - class ActionsClient(ActionsClientKit): def get_self_hosted_runner_group_for_org( self, diff --git a/runner_manager/dependencies.py b/runner_manager/dependencies.py index bfd6b72e..a8261703 100644 --- a/runner_manager/dependencies.py +++ b/runner_manager/dependencies.py @@ -1,7 +1,9 @@ +from datetime import timedelta from functools import lru_cache import httpx from githubkit.config import Config +from githubkit.retry import RetryRateLimit from redis import Redis from redis_om import get_redis_connection from rq import Queue @@ -37,11 +39,16 @@ def get_scheduler() -> Scheduler: @lru_cache() def get_github() -> GitHub: settings: Settings = get_settings() + auto_retry = RetryRateLimit( + max_retry=3 + ) if settings.github_auto_retry else None config: Config = Config( base_url=httpx.URL(str(settings.github_base_url)), follow_redirects=True, accept="*/*", user_agent="runner-manager", timeout=httpx.Timeout(30.0), + http_cache=True, + auto_retry=auto_retry, ) return GitHub(settings.github_auth_strategy(), config=config) diff --git a/runner_manager/jobs/workflow_job.py b/runner_manager/jobs/workflow_job.py index f70a5e74..7df28b57 100644 --- a/runner_manager/jobs/workflow_job.py +++ b/runner_manager/jobs/workflow_job.py @@ -1,14 +1,19 @@ from __future__ import annotations import logging -from datetime import timedelta -from githubkit.webhooks.models import ( - WorkflowJobCompleted, - WorkflowJobInProgress, - WorkflowJobQueued, +from datetime import timedelta, datetime + +from githubkit.versions.latest.models import ( + WebhookWorkflowJobCompleted as WorkflowJobCompleted, +) +from githubkit.versions.latest.models import ( + WebhookWorkflowJobInProgress as WorkflowJobInProgress, +) +from githubkit.versions.latest.models import ( + WebhookWorkflowJobQueued as WorkflowJobQueued, ) -from githubkit.webhooks.types import WorkflowJobEvent +from githubkit.versions.latest.webhooks import WorkflowJobEvent from runner_manager import Settings from runner_manager.clients.github import GitHub @@ -27,9 +32,15 @@ def log_workflow_job(webhook: WorkflowJobEvent) -> None: ) -def time_to_start(webhook: WorkflowJobInProgress | WorkflowJobCompleted) -> timedelta: +def time_to_start(webhook: WorkflowJobEvent) -> timedelta: """From a given webhook, calculate the time it took to start the job""" - return webhook.workflow_job.started_at - webhook.workflow_job.created_at + + assert isinstance(webhook.workflow_job.started_at, datetime) + assert isinstance(webhook.workflow_job.created_at, datetime) + return ( + webhook.workflow_job.started_at - webhook.workflow_job.created_at + ) + def completed(webhook: WorkflowJobCompleted) -> int: diff --git a/runner_manager/models/runner.py b/runner_manager/models/runner.py index 0b427d5e..3f0499fa 100644 --- a/runner_manager/models/runner.py +++ b/runner_manager/models/runner.py @@ -1,13 +1,12 @@ import logging from datetime import datetime, timedelta, timezone from enum import Enum -from typing import Dict, List, Literal, Optional +from typing import Dict, List, Literal, Optional, TypedDict import redis from githubkit.exception import RequestFailed -from githubkit.rest.models import Runner as GitHubRunner -from githubkit.rest.types import OrgsOrgActionsRunnersGenerateJitconfigPostBodyType -from githubkit.webhooks.types import WorkflowJobEvent +from githubkit.versions.latest.models import Runner as GitHubRunner +from githubkit.versions.latest.webhooks import WorkflowJobEvent from pydantic import BaseModel as PydanticBaseModel from redis_om import Field, NotFoundError @@ -70,7 +69,7 @@ class Runner(BaseModel): labels: List[RunnerLabel] = [] organization: str = Field(default=None, index=True, description="Organization name") created_at: Optional[datetime] - started_at: Optional[datetime] + started_at: Optional[datetime | str] def __str__(self): return f"{self.name} (status: {self.status}, busy: {self.busy})" @@ -182,11 +181,11 @@ def generate_jit_config(self, github: GitHub) -> "Runner": assert self.runner_group_id is not None, "Runner group id is required" jitconfig = github.rest.actions.generate_runner_jitconfig_for_org( org=self.organization, - data=OrgsOrgActionsRunnersGenerateJitconfigPostBodyType( - name=self.name, - runner_group_id=self.runner_group_id, - labels=[label.name for label in self.labels], - ), + data={ + "name": self.name, + "runner_group_id": self.runner_group_id, + "labels": [label.name for label in self.labels], + }, ).parsed_data self.id = jitconfig.runner.id self.encoded_jit_config = jitconfig.encoded_jit_config diff --git a/runner_manager/models/runner_group.py b/runner_manager/models/runner_group.py index ed210d19..c150a601 100644 --- a/runner_manager/models/runner_group.py +++ b/runner_manager/models/runner_group.py @@ -7,8 +7,10 @@ import redis from githubkit import Response from githubkit.exception import RequestFailed -from githubkit.webhooks.models import WorkflowJobInProgress -from githubkit.webhooks.types import WorkflowJobEvent +from githubkit.versions.latest.models import ( + WebhookWorkflowJobInProgress as WorkflowJobInProgress, +) +from githubkit.versions.latest.webhooks import WorkflowJobEvent from pydantic import BaseModel as PydanticBaseModel from pydantic import Field as PydanticField from pydantic import validator diff --git a/runner_manager/models/settings.py b/runner_manager/models/settings.py index 280d8718..30297849 100644 --- a/runner_manager/models/settings.py +++ b/runner_manager/models/settings.py @@ -43,6 +43,7 @@ class Settings(BaseSettings): github_installation_id: int = 0 github_client_id: Optional[str] = None github_client_secret: SecretStr = SecretStr("") + github_auto_retry: bool = True @property def app_install(self) -> bool: diff --git a/runner_manager/models/webhook.py b/runner_manager/models/webhook.py index 55fac262..85ba9789 100644 --- a/runner_manager/models/webhook.py +++ b/runner_manager/models/webhook.py @@ -1,6 +1,7 @@ from typing import Union -from githubkit.webhooks.types import PingEvent, WorkflowJobEvent +from githubkit.versions.latest.models import WebhookPing +from githubkit.versions.latest.webhooks import WorkflowJobEvent from pydantic import BaseModel @@ -10,4 +11,4 @@ class WebhookResponse(BaseModel): job_id: str | None = None -AcceptedWebhookEvents = Union[WorkflowJobEvent, PingEvent] +AcceptedWebhookEvents = Union[WorkflowJobEvent, WebhookPing] diff --git a/runner_manager/routers/webhook.py b/runner_manager/routers/webhook.py index 5bbf5aef..94c5bb2c 100644 --- a/runner_manager/routers/webhook.py +++ b/runner_manager/routers/webhook.py @@ -1,8 +1,8 @@ from typing import Annotated, Set from fastapi import APIRouter, Depends, Header, HTTPException, Security +from githubkit.versions.latest.models import WebhookPing from githubkit.webhooks import verify -from githubkit.webhooks.types import PingEvent from rq import Queue from runner_manager.dependencies import get_queue, get_settings @@ -43,7 +43,7 @@ def post( valid: bool = Security(validate_webhook), queue: Queue = Depends(get_queue), ) -> WebhookResponse: - if not isinstance(webhook, PingEvent): + if not isinstance(webhook, WebhookPing): action = webhook.action else: action = None diff --git a/tests/api/test_webhook_router.py b/tests/api/test_webhook_router.py index a58b224e..6d82345b 100644 --- a/tests/api/test_webhook_router.py +++ b/tests/api/test_webhook_router.py @@ -1,7 +1,7 @@ from functools import lru_cache from githubkit.webhooks import sign -from githubkit.webhooks.models import WorkflowJobCompleted +from githubkit.versions.latest.models import WebhookWorkflowJobCompleted as WorkflowJobCompleted from hypothesis import given from pytest import fixture diff --git a/tests/conftest.py b/tests/conftest.py index 035da134..91b6462c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,6 +56,8 @@ def github(settings, monkeypatch) -> GitHub: user_agent="runner-manager", follow_redirects=True, timeout=httpx.Timeout(5.0), + auto_retry=None, + http_cache=False ) monkeypatch.setattr(Paginator, "_get_next_page", get_next_monkeypatch) diff --git a/tests/strategies.py b/tests/strategies.py index effd6adb..3ed21167 100644 --- a/tests/strategies.py +++ b/tests/strategies.py @@ -1,21 +1,28 @@ from string import ascii_lowercase from typing import Optional -from githubkit.rest.models import Runner as GitHubRunner -from githubkit.webhooks.models import ( - License, - Organization, - PingEvent, - PingEventPropHook, - Repository, - User, - WorkflowJobCompleted, - WorkflowJobCompletedPropWorkflowJob, - WorkflowJobInProgress, - WorkflowJobInProgressPropWorkflowJob, - WorkflowJobQueued, - WorkflowJobQueuedPropWorkflowJob, - WorkflowStepCompleted, +from githubkit.versions.latest.models import JobPropStepsItems as WorkflowStepCompleted +from githubkit.versions.latest.models import License, Organization, Repository +from githubkit.versions.latest.models import Runner as GitHubRunner +from githubkit.versions.latest.models import SimpleUser as User +from githubkit.versions.latest.models import WebhookPing, WebhookPingPropHook +from githubkit.versions.latest.models import ( + WebhookWorkflowJobCompleted as WorkflowJobCompleted, +) +from githubkit.versions.latest.models import ( + WebhookWorkflowJobCompletedPropWorkflowJob as WorkflowJobCompletedPropWorkflowJob, +) +from githubkit.versions.latest.models import ( + WebhookWorkflowJobInProgress as WorkflowJobInProgress, +) +from githubkit.versions.latest.models import ( + WebhookWorkflowJobInProgressPropWorkflowJob as WorkflowJobInProgressPropWorkflowJob, +) +from githubkit.versions.latest.models import ( + WebhookWorkflowJobQueued as WorkflowJobQueued, +) +from githubkit.versions.latest.models import ( + WebhookWorkflowJobQueuedPropWorkflowJob as WorkflowJobQueuedPropWorkflowJob, ) from hypothesis import strategies as st from redis import Redis @@ -67,7 +74,7 @@ class Repo(Repository): runner_group_name=Text, runner_group_id=Int, labels=st.lists(Text, min_size=1, max_size=5), - started_at=st.datetimes(), + # started_at=str(st.datetimes()), ) JobPropInProgressStrategy = st.builds( @@ -79,8 +86,8 @@ class Repo(Repository): runner_group_name=Text, runner_group_id=Int, labels=st.lists(Text, min_size=1, max_size=5), - started_at=st.datetimes(), - created_at=st.datetimes(), + # started_at=st.text(st.datetimes()), + # created_at=str(st.datetimes()), ) JobPropQueuedStrategy = st.builds( @@ -146,7 +153,7 @@ class Repo(Repository): ) PingHookStrategy = st.builds( - PingEventPropHook, + WebhookPingPropHook, events=st.just(["*"]), ) @@ -157,4 +164,4 @@ class Repo(Repository): busy=st.booleans(), name=Text, ) -PingStrategy = st.builds(PingEvent, hook=PingHookStrategy) +PingStrategy = st.builds(WebhookPing, hook=PingHookStrategy) diff --git a/tests/unit/backend/test_gcp.py b/tests/unit/backend/test_gcp.py index d31a887f..139699a0 100644 --- a/tests/unit/backend/test_gcp.py +++ b/tests/unit/backend/test_gcp.py @@ -1,7 +1,7 @@ import os from typing import List -from githubkit.webhooks.types import WorkflowJobEvent +from githubkit.versions.latest.webhooks import WorkflowJobEvent from google.cloud.compute import Image, NetworkInterface from hypothesis import given from pytest import fixture, mark, raises diff --git a/tests/unit/jobs/test_workflow_job.py b/tests/unit/jobs/test_workflow_job.py index 15e786c1..bce4083c 100644 --- a/tests/unit/jobs/test_workflow_job.py +++ b/tests/unit/jobs/test_workflow_job.py @@ -2,10 +2,14 @@ from time import sleep from uuid import uuid4 -from githubkit.webhooks.models import ( - WorkflowJobCompleted, - WorkflowJobInProgress, - WorkflowJobQueued, +from githubkit.versions.latest.models import ( + WebhookWorkflowJobCompleted as WorkflowJobCompleted, +) +from githubkit.versions.latest.models import ( + WebhookWorkflowJobInProgress as WorkflowJobInProgress, +) +from githubkit.versions.latest.models import ( + WebhookWorkflowJobQueued as WorkflowJobQueued, ) from hypothesis import assume, given, settings from redis import Redis diff --git a/tests/unit/models/test_runner.py b/tests/unit/models/test_runner.py index d420b52b..05d453a6 100644 --- a/tests/unit/models/test_runner.py +++ b/tests/unit/models/test_runner.py @@ -1,7 +1,9 @@ from datetime import datetime, timezone import pytest -from githubkit.webhooks.models import WorkflowJobCompleted +from githubkit.versions.latest.models import ( + WebhookWorkflowJobCompleted as WorkflowJobCompleted, +) from hypothesis import given from hypothesis import strategies as st from redis_om import Migrator, NotFoundError diff --git a/tests/unit/models/test_runner_group.py b/tests/unit/models/test_runner_group.py index d86f5661..78bcc4a9 100644 --- a/tests/unit/models/test_runner_group.py +++ b/tests/unit/models/test_runner_group.py @@ -1,7 +1,9 @@ from datetime import timezone import pytest -from githubkit.webhooks.models import WorkflowJobCompleted +from githubkit.versions.latest.models import ( + WebhookWorkflowJobCompleted as WorkflowJobCompleted, +) from hypothesis import given from pydantic import ValidationError from redis_om import Migrator, NotFoundError