PTFE-584 bootstrap project design and structure

.devcontainer/devcontainer.json

@@ -3,22 +3,38 @@
   "image": "mcr.microsoft.com/devcontainers/base:1-jammy",
   "features": {
     "ghcr.io/devcontainers/features/python": {
-      "version": "3.11"
+      "version": "3.11",
+      "installTools": false
     },
     "ghcr.io/devcontainers/features/docker-in-docker": {},
     "ghcr.io/devcontainers/features/github-cli:1": {},
     "ghcr.io/devcontainers/features/sshd:1": {},
     "ghcr.io/devcontainers-contrib/features/poetry:2": {},
-    "ghcr.io/trunk-io/devcontainer-feature/trunk:1": {}
+    "ghcr.io/trunk-io/devcontainer-feature/trunk:1": {},
+    "ghcr.io/devcontainers/features/common-utils:2": {
+      "configureZshAsDefaultShell": true,
+      "username": "vscode",
+      "uid": 1000,
+      "gid": 1000
+    }
   },
   "customizations": {
     "vscode": {
-      "extensions": ["bungcip.better-toml", "GitHub.copilot"]
+      "extensions": [
+        "GitHub.copilot",
+        "bierner.markdown-mermaid",
+        "tamasfe.even-better-toml"
+      ],
+      "settings": {
+        "python.analysis.typeCheckingMode": "basic",
+        "python.defaultInterpreterPath": ".venv/bin/python"
+      }
     }
   },
+  "remoteUser": "vscode",
   "postCreateCommand": "poetry install --no-interaction --no-ansi --no-root",
-  "postStartCommand": {
-    "compose": "docker compose up --build",
+  "postAttachCommand": {
+    "compose": "docker compose --profile develop up --build",
     "watch": "docker compose alpha watch"
   },
   "portsAttributes": {
@@ -29,6 +45,14 @@
     "8001": {
       "label": "redis-insight",
       "onAutoForward": "openPreview"
+    },
+    "8000": {
+      "label": "runner-manager",
+      "onAutoForward": "silent"
+    },
+    "4010": {
+      "label": "github-mock",
+      "onAutoForward": "silent"
     }
   }
 }

.dockerignore

@@ -1 +1,8 @@
 .git
+tests
+.devcontainer
+.github
+.pytest_cache
+.trunk
+.venv
+.hypothesis

.github/workflows/tests.yaml

@@ -0,0 +1,25 @@
+---
+name: tests
+
+on: pull_request
+
+permissions:
+  contents: read
+
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis/redis-stack
+        ports:
+          - 6379:6379
+    steps:
+      - uses: actions/checkout@v3
+      - run: pipx install poetry
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: poetry
+      - run: poetry install
+      - run: poetry run pytest tests/unit

.gitignore

@@ -1,2 +1,4 @@
 __pycache__
 .venv
+.pytest_cache
+.hypothesis

.trunk/.gitignore

@@ -5,6 +5,4 @@
 plugins
 user_trunk.yaml
 user.yaml
-# TODO(lauri): Remove shims after a release or two
-shims
 tools

.trunk/configs/.markdownlint.yaml

@@ -8,3 +8,5 @@ line_length: false
 spaces: false
 url: false
 whitespace: false
+
+MD046: false

.trunk/trunk.yaml

@@ -1,30 +1,39 @@
 version: 0.1
 cli:
-  version: 1.12.0
+  version: 1.13.0
 plugins:
   sources:
     - id: trunk
-      ref: v0.0.22
+      ref: v1.0.0
       uri: https://github.com/trunk-io/plugins
 lint:
   enabled:
     - [email protected]
     - [email protected]
-    - [email protected].278
-    - taplo@0.7.0
+    - [email protected].280
+    - taplo@0.8.1
     - [email protected]
-    - [email protected].327
+    - [email protected].340
     - git-diff-check
     - [email protected]
     - [email protected]
     - [email protected]
     - [email protected]
-    - [email protected]
+    - [email protected]
+    - [email protected]
+  disabled:
+    - bandit
+    - osv-scanner
+    - trivy
+    - terrascan
 runtimes:
   enabled:
     - [email protected]
     - [email protected]
 actions:
+  disabled:
+    - trunk-check-pre-push
+    - trunk-fmt-pre-commit
   enabled:
     - trunk-announce
     - trunk-upgrade-available

Dockerfile

@@ -1,18 +1,26 @@
 FROM python:3.11-slim
 
 EXPOSE 8000
+HEALTHCHECK NONE
 
 # install poetry
 ARG POETRY_VERSION=1.5.1
 ENV POETRY_VERSION=${POETRY_VERSION} \
     POETRY_NO_INTERACTION=1 \
     POETRY_NO_ANSI=1
 
-RUN pip install poetry==$POETRY_VERSION
+RUN pip install --no-cache-dir poetry==$POETRY_VERSION
+
+# Create a runner-manager group and user
+RUN groupadd -r runner-manager && \
+    useradd --no-log-init -r -g runner-manager runner-manager && \
+    mkdir -p /home/runner-manager && \
+    chown -R runner-manager:runner-manager /home/runner-manager
 
 # copy project requirement files here to ensure they will be cached.
 WORKDIR /app
-COPY poetry.lock pyproject.toml /app/
+
+COPY --chown=runner-manager:runner-manager poetry.lock pyproject.toml /app/
 
 # install project dependencies
 RUN poetry config virtualenvs.create false \
@@ -21,5 +29,7 @@ RUN poetry config virtualenvs.create false \
 # copy project
 COPY . /app
 
+USER runner-manager
+
 # run entrypoint.sh
-CMD ["uvicorn", "runner_manager.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "runner_manager.main:app", "--host", "0.0.0.0", "--port", "8000"]

docker-compose.yaml

@@ -15,15 +15,32 @@ services:
       - 8000:8000
     environment:
       REDIS_OM_URL: redis://redis:6379
+    command: uvicorn runner_manager.main:app --host 0.0.0.0 --port 8000 --reload
+    volumes:
+      - ./runner_manager:/app/runner_manager
     x-develop:
       watch:
-        - action: sync
-          path: ./runner_manager
-          target: /app/runner_manager
         - action: rebuild
-          path: .
-          target: /app
-
+          path: poetry.lock
+        - action: rebuild
+          path: docker-compose.yaml
+  worker:
+    build: .
+    command: rq worker -c runner_manager.jobs.settings
+    volumes:
+      - .:/app
+    environment:
+      REDIS_OM_URL: redis://redis:6379
+  github-mock:
+    profiles:
+      - develop
+    build: tests/images/github-mock
+    ports:
+      - 4010:4010
+    x-develop:
+      watch:
+        - action: rebuild
+          path: tests/images/github-mock
 
 volumes:
   redis_data:

docs/development/code.md

@@ -0,0 +1,39 @@
+# Code
+
+The Runner Manager is written in Python, typing will be used and enforced
+by [pyright](https://github.com/microsoft/pyright).
+
+All libraries must be compatible with [Pydantic](https://docs.pydantic.dev/).
+This requirement will allow to have a nice developer experience
+especially when combined with [Fastapi](#fastapi).
+
+## Dependencies
+
+The main dependencies are described below along with the reason why they were chosen.
+
+### Fastapi
+
+[Fastapi](https://fastapi.tiangolo.com/) will be used as a web framework to
+expose the API of the runner manager.
+
+### GitHubKit
+
+[GitHubKit](https://github.com/yanyongyu/githubkit) is a GitHub API client for
+Python that works exactly like the official GitHub API client for JavaScript, Octokit.
+
+Its code is generated from the OpenAPI specification of the GitHub API.
+
+### Redis-om
+
+[Redis-om](https://github.com/redis/redis-om-python) is a Redis Object Mapper
+that allows to store and retrieve Python Pydantic objects in Redis.
+
+This enable fastapi objects to be stored in Redis and retrieved
+without having to write boilerplate code.
+
+### RQ
+
+[RQ](https://python-rq.org/) is a simple Python library for queueing jobs and
+processing them in the background with workers.
+
+It will be used to process the jobs that will be created by the runner manager.

docs/development/concepts.md

@@ -0,0 +1,71 @@
+# Concepts
+
+Described below are the concepts that are used in the runner manager.
+They should help to understand the rest of the documentation.
+
+## Runners
+
+A runner is a machine that is used to run jobs in [GitHub Actions].
+By default, GitHub provides runners that are hosted by them, they
+are called [GitHub-hosted runners].
+
+[GitHub Actions] also supports [self-hosted runners], the difference
+being that they are managed and hosted by the user.
+They will be the focus of the runner manager, and will be referred
+to as runners.
+
+## Runner groups
+
+The [runner groups] are a way to organize runners in [GitHub Actions].
+In the runner manager, runner groups are used to configure
+the specification of the runners that will be created.
+
+Each group will be matched with its appropriate runner group
+in the GitHub organization or repository.
+
+!!! note
+
+    We use the term [runner groups] for consistency with [GitHub Actions] API.
+
+More information about the [runner groups] configuration can be found in the
+[configuration documentation](./configuration.md#runner-groups).
+
+## Backends
+
+The runner manager supports multiple backends, which are responsible
+for hosting the runners. The following backends will be supported:
+
+- GCP.
+- AWS.
+- Docker (For local functional testing).
+- FakeBackend (For local unit testing).
+
+## Database
+
+The runner manager is stateful and needs to gather information that comes from both
+GitHub and the backend that has been configured.
+
+This data needs ideally to be persistent, so that the runner manager can
+recover from a reboot or crash and not lose track of the runners
+that are currently running.
+
+Redis will be used as a database to store the state of the runners.
+
+## Jobs
+
+The runner manager will be responsible for jobs that are triggered
+by events coming from GitHub as well as health checks that are
+triggered periodically.
+
+A proper queue needs to be used to ensure that tasks are:
+
+- Not lost if the runner manager crashes.
+- Properly distributed between multiple runner manager instances.
+
+Redis is used as a task queue backend, so that only one database server
+is required.
+
+[Runner groups]: https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/managing-self-hosted-runners/managing-access-to-self-hosted-runners-using-groups#about-runner-groups
+[GitHub-hosted runners]: https://docs.github.com/en/enterprise-cloud@latest/actions/using-github-hosted-runners/about-github-hosted-runners
+[self-hosted runners]: https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners
+[GitHub Actions]: https://docs.github.com/en/actions