Merge branch 'main' into feature/PTFE-700-add-build-workflow

.devcontainer/devcontainer.json

@@ -1,6 +1,9 @@
 {
   "name": "Python 3",
   "image": "mcr.microsoft.com/devcontainers/base:1-jammy",
+  "remoteEnv": {
+    "GITHUB_BASE_URL": "http://localhost:4010"
+  },
   "features": {
     "ghcr.io/devcontainers/features/python": {
       "version": "3.11",
@@ -23,11 +26,15 @@
       "extensions": [
         "GitHub.copilot",
         "bierner.markdown-mermaid",
-        "tamasfe.even-better-toml"
+        "tamasfe.even-better-toml",
+        "github.vscode-github-actions"
       ],
       "settings": {
         "python.analysis.typeCheckingMode": "basic",
-        "python.defaultInterpreterPath": ".venv/bin/python"
+        "python.defaultInterpreterPath": ".venv/bin/python",
+        "python.testing.pytestArgs": ["tests"],
+        "python.testing.unittestEnabled": false,
+        "python.testing.pytestEnabled": true
       }
     }
   },

.github/dependabot.yaml

@@ -0,0 +1,33 @@
+---
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: daily
+    commit-message:
+      prefix: "github-actions"
+      include: "scope"
+    labels:
+      - "github-actions"
+      - "dependencies"
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: daily
+    commit-message:
+      prefix: "docker"
+      include: "scope"
+    labels:
+      - "docker"
+      - "dependencies"
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: daily
+    commit-message:
+      prefix: "pip"
+      include: "scope"
+    labels:
+      - "pip"
+      - "dependencies"

.github/workflows/codeql.yaml

@@ -0,0 +1,24 @@
+---
+name: codeQL
+
+on:
+    push:
+      branches: [main]
+    pull_request:
+      branches: [main]
+    workflow_dispatch: {}
+
+jobs:
+    analyze:
+      name: Static analysis with CodeQL
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v3
+        - name: Initialize CodeQL
+          uses: github/codeql-action/init@v2
+          with:
+            languages: python
+
+        - name: Build and analyze
+          uses: github/codeql-action/analyze@v2

.github/workflows/dependency-review.yaml

@@ -0,0 +1,15 @@
+---
+name: dependency review
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+      - name: Dependency Review
+        uses: actions/dependency-review-action@v3

.github/workflows/release.yaml

@@ -0,0 +1,36 @@
+name: release
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: Tag to be released
+        required: true
+      prerelease:
+        description: Define the release as pre-release
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  docker:
+    uses: scality/workflows/.github/workflows/docker-build.yaml@v1
+    with:
+      name: runner-manager
+      namespace: scality
+      tag: ${{ inputs.tag }}
+
+  release:
+    runs-on: ubuntu-latest
+    needs: docker
+    steps:
+      - name: Create Release
+        uses: softprops/action-gh-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          target_commitish: ${{ github.sha }}
+          tag_name: ${{ inputs.tag }}
+          name: Release ${{ inputs.tag }}
+          prerelease: ${{ inputs.prerelease }}
+          generate_release_notes: true

.github/workflows/test-deployment.yaml

@@ -0,0 +1,46 @@
+---
+name: test-deployment
+
+
+on: pull_request
+
+permissions:
+  contents: read
+
+jobs:
+  docker-build:
+    permissions:
+      contents: read
+      packages: write
+    uses: scality/workflows/.github/workflows/docker-build.yaml@v1
+    with:
+      name: runner-manager
+      namespace: scality
+  test-deployment:
+    needs: docker-build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: helm/kind-action@v1
+      - name: Edit the runner-manager image
+        working-directory: tests
+        run: |
+          cat <<EOF > kustomization.yaml
+          apiVersion: kustomize.config.k8s.io/v1beta1
+          kind: Kustomization
+          images:
+            - name: ghcr.io/scality/runner-manager
+              newTag: ${{ github.sha }}
+          resources:
+            - ../manifests
+          EOF
+      - name: Deploy runner-manager
+        run: kustomize build tests/ | kubectl apply -f -
+      - name: Check if deployment is ready
+        run: |
+          kubectl rollout status statefulset redis --timeout=90s
+          kubectl rollout status deployment runner-manager --timeout=120s
+          kubectl rollout status deployment runner-manager-worker --timeout=120s
+      - name: get all resources
+        if: failure()
+        run: kubectl get all

.github/workflows/tests.yaml

@@ -7,19 +7,29 @@ permissions:
   contents: read
 
 jobs:
-  unit:
+  test:
+    name: ${{ matrix.test }}
     runs-on: ubuntu-latest
-    services:
-      redis:
-        image: redis/redis-stack
-        ports:
-          - 6379:6379
+    strategy:
+      matrix:
+        test: [unit, api]
+
+    env:
+      REDIS_OM_URL: redis://localhost:6379
+      GITHUB_BASE_URL: http://localhost:4010
+
     steps:
       - uses: actions/checkout@v3
+      - name: Boot compose services
+        run: docker compose --profile tests up --build --detach
       - run: pipx install poetry
       - uses: actions/setup-python@v4
         with:
           python-version: 3.11
           cache: poetry
       - run: poetry install
-      - run: poetry run pytest tests/unit
+      - name: Run tests
+        run: poetry run pytest tests/${{ matrix.test }}
+      - name: Dump logs
+        run: docker compose --profile tests logs
+        if: always()

.github/workflows/trunk-check.yaml

@@ -19,6 +19,5 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-
       - name: Trunk Check
         uses: trunk-io/trunk-action@v1

.trunk/trunk.yaml

@@ -20,18 +20,12 @@ lint:
     - [email protected]
     - [email protected]
     - [email protected]
-    - [email protected]
   disabled:
+    - checkov
     - bandit
     - osv-scanner
     - trivy
     - terrascan
-  triggers:
-    - linters: [checkov]
-      paths:
-        - .github/workflows
-      targets:
-        - .github/workflows
 runtimes:
   enabled:
     - [email protected]

docker-compose.yaml

@@ -3,13 +3,20 @@ version: "2"
 
 services:
   redis:
+    profiles:
+      - main
+      - develop
+      - tests
     image: redis/redis-stack
     ports:
       - 6379:6379
       - 8001:8001
     volumes:
       - redis_data:/data
   runner-manager:
+    profiles:
+      - main
+      - develop
     build: .
     ports:
       - 8000:8000
@@ -25,6 +32,9 @@ services:
         - action: rebuild
           path: docker-compose.yaml
   worker:
+    profiles:
+      - main
+      - develop
     build: .
     command: rq worker -c runner_manager.jobs.settings
     volumes:
@@ -34,6 +44,7 @@ services:
   github-mock:
     profiles:
       - develop
+      - tests
     build: tests/images/github-mock
     ports:
       - 4010:4010

docs/development/configuration.md

@@ -14,6 +14,8 @@ The global configuration will contain the following information:
 - A name for the runner manager. (default: runner-manager)
   It will be used as a metadata or prefix which will allow the runner to identify
   the owner of each resources, avoiding conflicts with other runner managers or users.
+- The default GitHub organization in which the runner and groups will be created.
+  (Required)
 - The GitHub Authentication parameters. (Required)
   - For GitHub Application: The GitHub Application ID, Installation ID and Private Key. (Required)
   - For GitHub Personal Access Token: The GitHub Personal Access Token. (Required)
@@ -32,7 +34,7 @@ settings file.
 The following information will be configured for each runner group:
 
 - Name of the group. (Required)
-- Name of the GitHub Organization in which the runner and group will be created. (Required)
+- Name of the GitHub Organization in which the runner and group will be created. (optional)
 - Name of the GitHub Repository in which the runner and group will be created. (Optional)
 - Repository access:
   - A list of selected repositories. (Default: All repositories)
@@ -44,7 +46,6 @@ The following information will be configured for each runner group:
   octo-org/octo-repo/.github/workflows/deploy.yml@d6dc6c96df4f32fa27b039f2084f576ed2c5c2a5,
   monalisa/octo-test/.github/workflows/test.yml@main
   ```
-- The name of the workflow that will be used to run the jobs. (Optional)
 - The maximum number of runners that can run simultaneously. (Default: 20)
 - The minimum number of runners that must be available. (Default: 0)
 - The runner labels that will be attached to the runners of the group. (Required)

manifests/base/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - runner-manager
+  - redis

manifests/base/redis/kustomization.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+
+commonLabels:
+  app.kubernetes.io/instance: redis
+  app.kubernetes.io/name: redis
+  app.kubernetes.io/component: database
+  app.kubernetes.io/part-of: runner-manager
+images:
+  - name: redis
+    newName: redis/redis-stack
+    newTag: 6.2.6-v9
+resources:
+  - statefulset.yaml
+  - service.yaml

manifests/base/redis/service.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+spec:
+  clusterIP: None
+  ports:
+    - protocol: TCP
+      port: 6379
+      targetPort: 6379