Skip to content

Commit

Permalink
api: add metrics endpoint (#630)
Browse files Browse the repository at this point in the history
Adds a metrics endpoint that computes updated metric values on each
request.

On each request we:
- fetch runner groups from Redis
- get all runners for each group
- update a Gauge for the runners_count on each group
- render out the prometheus metrics in standard format

Resolves #625
  • Loading branch information
lewismiddleton authored Jul 11, 2024
1 parent 94e45c1 commit 86ae1ba
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 3 deletions.
18 changes: 16 additions & 2 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ rq-scheduler = "^0.13.1"
pyvmomi = "^8.0.2.0.1"
vapi-runtime = { url = "https://raw.githubusercontent.com/vmware/vsphere-automation-sdk-python/v8.0.1.0/lib/vapi-runtime/vapi_runtime-2.40.0-py2.py3-none-any.whl" }
vcenter-bindings = { url = "https://raw.githubusercontent.com/vmware/vsphere-automation-sdk-python/v8.0.1.0/lib/vcenter-bindings/vcenter_bindings-4.1.0-py2.py3-none-any.whl" }
prometheus-client = "^0.20.0"


[tool.poetry.group.docs]
Expand Down
10 changes: 9 additions & 1 deletion runner_manager/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,14 @@
from runner_manager import Runner, RunnerGroup, Settings, log
from runner_manager.dependencies import get_queue, get_redis, get_settings
from runner_manager.jobs.startup import startup
from runner_manager.routers import _health, private, public, runner_groups, webhook
from runner_manager.routers import (
_health,
metrics,
private,
public,
runner_groups,
webhook,
)

settings = get_settings()
log.setLevel(settings.log_level)
Expand Down Expand Up @@ -39,6 +46,7 @@ async def lifespan(app: FastAPI):
app.include_router(private.router)
app.include_router(public.router)
app.include_router(runner_groups.router)
app.include_router(metrics.router)


@app.get("/")
Expand Down
22 changes: 22 additions & 0 deletions runner_manager/routers/metrics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
from typing import List

from fastapi import APIRouter
from fastapi.responses import PlainTextResponse
from prometheus_client import Gauge, generate_latest

from runner_manager import RunnerGroup
from runner_manager.models.runner import Runner

router = APIRouter(prefix="/metrics")

runners_count = Gauge("runners_count", "Number of runners", ["runner_group"])


@router.get("/", response_class=PlainTextResponse)
def compute_metrics() -> PlainTextResponse:
groups: List[RunnerGroup] = RunnerGroup.find().all()
for group in groups:
runners: List[Runner] = group.get_runners()
runners_count.labels(runner_group=group.name).set(len(runners))
metrics = generate_latest().decode()
return PlainTextResponse(content=metrics)
65 changes: 65 additions & 0 deletions tests/api/test_metrics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
from fastapi.testclient import TestClient

from runner_manager import RunnerGroup
from runner_manager.clients.github import GitHub


def test_metrics_endpoint(client: TestClient, runner_group: RunnerGroup):
runner_group.save()
response = client.get("/metrics")
assert response.status_code == 200
runner_lines = [
line for line in response.text.splitlines() if line.startswith("runners_")
]
assert f'runners_count{{runner_group="{runner_group.name}"}} 0.0' in runner_lines


def test_runner_count(client: TestClient, runner_group: RunnerGroup, github: GitHub):
runner_group.save()
want = len(runner_group.get_runners())

response = client.get("/metrics")
assert response.status_code == 200
got = [line for line in response.text.splitlines() if line.startswith("runners_")]
print(want)
assert f'runners_count{{runner_group="{runner_group.name}"}} {want:.1f}' in got

runner_group.max = 2
runner_group.min = 1
runner_group.save()

want = 0.0
response = client.get("/metrics")
assert response.status_code == 200
before = [
line for line in response.text.splitlines() if line.startswith("runners_")
]
assert f'runners_count{{runner_group="{runner_group.name}"}} {want:.1f}' in before

runner = runner_group.create_runner(github)
assert runner is not None

want = 1.0
response = client.get("/metrics")
assert response.status_code == 200
after_create = [
line for line in response.text.splitlines() if line.startswith("runners_")
]
assert (
f'runners_count{{runner_group="{runner_group.name}"}} {want:.1f}'
in after_create
)

runner = runner_group.delete_runner(runner, github)
assert runner is not None

want = 0.0
response = client.get("/metrics")
assert response.status_code == 200
after_delete = [
line for line in response.text.splitlines() if line.startswith("runners_")
]
assert (
f'runners_count{{runner_group="{runner_group.name}"}} {want:.1f}'
in after_delete
)

0 comments on commit 86ae1ba

Please sign in to comment.