added glitchtip service to manage issues from errors in core-engine

swiss-ai-center · Dec 5, 2023 · 1a7dadf · 1a7dadf
1 parent 5a3fb20
commit 1a7dadf
Show file tree

Hide file tree

Showing 12 changed files with 307 additions and 3 deletions.
diff --git a/.github/workflows/core-engine.yml b/.github/workflows/core-engine.yml
@@ -102,6 +102,7 @@ jobs:
           S3_HOST: ${{ secrets.ENGINE_DEV_S3_HOST }}
           S3_REGION: eu-central-2
           S3_BUCKET: engine
+          SENTRY_DSN: ${{ secrets.GLITCHTIP_SENTRY_DSN }}
         run: |
           # Set Core Engine version
           docker_image_tags=(${{ needs.release.outputs.docker-image-tags }})

diff --git a/.github/workflows/glitchtip.yml b/.github/workflows/glitchtip.yml
@@ -0,0 +1,80 @@
+# Documentation: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsuses
+name: glitchtip-workflow
+run-name: glitchtip workflow
+
+# Allow one concurrent deployment
+concurrency:
+  group: "glitchtip"
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - main
+      - prod
+    paths:
+      - .github/actions/execute-command-on-kubernetes-cluster/action.yml
+      - .github/workflows/glitchtip.yml
+      - core-engine/kubernetes/glitchtip.config-map.yml
+      - core-engine/kubernetes/glitchtip.deployment.yml
+      - core-engine/kubernetes/glitchtip.ingress.yml
+      - core-engine/kubernetes/glitchtip.service.yml
+      - core-engine/kubernetes/glitchtip.pvc.yml
+
+  pull_request:
+    paths:
+      - .github/actions/execute-command-on-kubernetes-cluster/action.yml
+      - .github/workflows/glitchtip.yml
+      - core-engine/kubernetes/glitchtip.config-map.yml
+      - core-engine/kubernetes/glitchtip.deployment.yml
+      - core-engine/kubernetes/glitchtip.ingress.yml
+      - core-engine/kubernetes/glitchtip.service.yml
+      - core-engine/kubernetes/glitchtip.pvc.yml
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    # Only run on main
+    if: success() && github.ref == 'refs/heads/main'
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+
+      - name: Prepare configuration files
+        shell: bash
+        working-directory: glitchtip/kubernetes
+        env:
+          HOST: https://monitor-swiss-ai-center.kube.isc.heia-fr.ch
+          ENABLE_USER_REGISTRATION: False
+          DATABASE_URL: ${{ secrets.GLITCHTIP_DATABASE_URL }}
+          SECRET_KEY: ${{ secrets.GLITCHTIP_SECRET_KEY }}
+          PORT: "8000"
+          EMAIL_URL: ${{ secrets.GLITCHTIP_SMTP_CONFIG }}
+          GLITCHTIP_DOMAIN: https://monitor-swiss-ai-center.kube.isc.heia-fr.ch
+          DEFAULT_FROM_EMAIL: "[email protected]"
+          CELERY_WORKER_AUTOSCALE: "1,3"
+          CELERY_WORKER_MAX_TASKS_PER_CHILD: "10000"
+        run: |
+          # Set Glitchtip configuration (ConfigMap)
+          yq '.data = (.data | to_entries | map({"key": .key, "value": "${" + .key + "}"}) | from_entries)' glitchtip.config-map.yml | envsubst > new-glitchtip.config-map.yml && mv new-glitchtip.config-map.yml glitchtip.config-map.yml
+
+          # Set Glitchtip configuration (Ingress)
+          yq ".spec.rules[0].host = \"${HOST#*://}\"" glitchtip.ingress.yml > new-glitchtip.ingress.yml && mv new-glitchtip.ingress.yml glitchtip.ingress.yml
+          yq ".spec.tls[0].hosts[0] = \"${HOST#*://}\"" glitchtip.ingress.yml > new-glitchtip.ingress.yml && mv new-glitchtip.ingress.yml glitchtip.ingress.yml
+
+      - name: Deploy Glitchtip on the Kubernetes cluster
+        uses: ./.github/actions/execute-command-on-kubernetes-cluster
+        with:
+          kube-config: ${{ secrets.KUBE_CONFIG_DEV }}
+          kube-namespace: swiss-ai-center-dev
+          kubectl-context: ./glitchtip/kubernetes
+          kubectl-args: |
+            apply \
+              -f glitchtip.config-map.yml \
+              -f glitchtip.pvc.yml \
+              -f glitchtip.deployment.yml \
+              -f glitchtip.service.yml \
+              -f glitchtip.ingress.yml
diff --git a/core-engine/.env b/core-engine/.env
@@ -33,3 +33,6 @@ S3_BUCKET=engine
 
 # The inverval (in seconds) to check the services availability
 CHECK_SERVICES_AVAILABILITY_INTERVAL=30
+
+# DSN for Error reporting
+SENTRY_DSN=""
diff --git a/core-engine/kubernetes/glitchtip.config-map.yml b/core-engine/kubernetes/glitchtip.config-map.yml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: glitchtip-config
+  labels:
+    app: glitchtip
+data:
+  ENABLE_USER_REGISTRATION: False
+  DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres
+  SECRET_KEY: change_me
+  PORT: "8000"
+  EMAIL_URL: "smtp://email:password@smtp_url:465"
+  GLITCHTIP_DOMAIN: http://localhost:8000
+  DEFAULT_FROM_EMAIL: "[email protected]"
+  CELERY_WORKER_AUTOSCALE: "1,3"
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: "10000"
+  POSTGRES_DB: postgres
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: postgres
diff --git a/core-engine/kubernetes/glitchtip.deployment.yml b/core-engine/kubernetes/glitchtip.deployment.yml
@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: glitchtip
+  labels:
+    app: glitchtip
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: glitchtip
+  template:
+    metadata:
+      labels:
+        app: glitchtip
+    spec:
+      containers:
+        - name: glitchtip
+          image: glitchtip/glitchtip
+          ports:
+            - containerPort: 8000
+          envFrom:
+            - configMapRef:
+                name: glitchtip-config
+          volumeMounts:
+            - name: uploads
+              mountPath: /code/uploads
+        - name: glitchtip-worker
+          image: glitchtip/glitchtip
+          command: ./bin/run-celery-with-beat.sh
+          envFrom:
+            - configMapRef:
+                name: glitchtip-config
+          volumeMounts:
+            - name: uploads
+              mountPath: /code/uploads
+        - name: glitchtip-migrate
+          image: glitchtip/glitchtip
+          command: ./manage.py migrate
+          envFrom:
+            - configMapRef:
+                name: glitchtip-config
+          volumeMounts:
+            - name: uploads
+              mountPath: /code/uploads
+        - name: redis
+          image: redis
+      volumes:
+        - name: uploads
+          persistentVolumeClaim:
+            claimName: uploads-pvc
+
diff --git a/core-engine/kubernetes/glitchtip.ingress.yml b/core-engine/kubernetes/glitchtip.ingress.yml
@@ -0,0 +1,20 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: glitchtip-ingress
+  namespace: pet-via-adv-da-ba23
+spec:
+  rules:
+    - host: monitor-swiss-ai-center.kube.isc.heia-fr.ch
+      http:
+        paths:
+          - backend:
+              service:
+                name: glitchtip
+                port:
+                  number: 80
+            path: /
+            pathType: Prefix
+  tls:
+    - hosts:
+        - monitor-swiss-ai-center.kube.isc.heia-fr.ch
diff --git a/core-engine/kubernetes/glitchtip.pvc.yml b/core-engine/kubernetes/glitchtip.pvc.yml
@@ -0,0 +1,10 @@
+apiVersion: apps/v1
+kind: PersistentVolumeClaim
+metadata:
+  name: uploads-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
diff --git a/core-engine/kubernetes/glitchtip.service.yml b/core-engine/kubernetes/glitchtip.service.yml
@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Service
+metadata:
+  name: glitchtip
+spec:
+  selector:
+    app: glitchtip
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8000
+---
+apiVersion: apps/v1
+kind: Service
+metadata:
+  name: postgres
+spec:
+  selector:
+    app: glitchtip
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432
+---
+apiVersion: apps/v1
+kind: Service
+metadata:
+  name: redis
+spec:
+  selector:
+    app: glitchtip
+  ports:
+    - protocol: TCP
+      port: 6379
+      targetPort: 6379
diff --git a/core-engine/requirements-all.txt b/core-engine/requirements-all.txt
@@ -68,3 +68,73 @@ websockets==11.0.3
 Werkzeug==2.3.6
 wrapt==1.15.0
 yarl==1.9.2
+aiobotocore==2.4.1
+aiofile==3.8.7
+aiofiles==22.1.0
+aiohttp==3.8.4
+aioitertools==0.11.0
+aiosignal==1.3.1
+anyio==3.7.0
+async-timeout==4.0.2
+attrs==23.1.0
+botocore==1.27.59
+caio==0.9.12
+certifi==2023.5.7
+charset-normalizer==3.1.0
+click==8.1.3
+common-code @ git+https://github.com/swiss-ai-center/core-engine.git@f590aee438b4fa1cca5936433f25159f61025c07#subdirectory=common-code
+coverage==7.2.7
+dnspython==2.3.0
+docker==6.1.3
+email-validator==1.3.0
+exceptiongroup==1.1.1
+fastapi==0.87.0
+flake8==5.0.4
+frozenlist==1.3.3
+greenlet==2.0.2
+h11==0.14.0
+httpcore==0.16.3
+httpx==0.23.1
+idna==3.4
+iniconfig==2.0.0
+jmespath==1.0.1
+makefun==1.15.0
+MarkupSafe==2.1.3
+mccabe==0.7.0
+minio==7.1.15
+multidict==6.0.4
+packaging==23.1
+pluggy==1.2.0
+psycopg2-binary==2.9.5
+pycodestyle==2.9.1
+pydantic==1.10.9
+pyflakes==2.5.0
+pytest==7.2.0
+pytest-asyncio==0.20.3
+pytest-cov==4.0.0
+pytest-httpserver==1.0.6
+python-dateutil==2.8.2
+python-dotenv==0.21.0
+python-multipart==0.0.5
+PyYAML==6.0
+requests==2.31.0
+rfc3986==1.5.0
+sentry-sdk==1.37.1
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==1.4.41
+sqlalchemy2-stubs==0.0.2a34
+sqlmodel==0.0.8
+starlette==0.21.0
+testcontainers-core==0.0.1rc1
+testcontainers-minio==0.0.1rc1
+testcontainers-postgres==0.0.1rc1
+tomli==2.0.1
+typing_extensions==4.6.3
+urllib3==1.26.16
+uvicorn==0.19.0
+websocket-client==1.6.1
+websockets==11.0.3
+Werkzeug==2.3.6
+wrapt==1.15.0
+yarl==1.9.2
diff --git a/core-engine/requirements.txt b/core-engine/requirements.txt
@@ -8,6 +8,7 @@ pytest-cov==4.0.0
 pytest-httpserver==1.0.6
 psycopg2-binary==2.9.5
 python-multipart==0.0.5
+sentry-sdk[fastapi]==1.37.1
 sqlmodel==0.0.8
 testcontainers-core==0.0.1rc1
 testcontainers-minio==0.0.1rc1

diff --git a/core-engine/src/config.py b/core-engine/src/config.py
@@ -29,6 +29,7 @@ class Settings(BaseSettings):
     s3_host: str
     s3_bucket: str
     check_services_availability_interval: int = 30
+    sentry_dsn: str
 
     class Config:
         env_file = ".env", "../.env"

diff --git a/core-engine/src/main.py b/core-engine/src/main.py
@@ -20,20 +20,33 @@
 from database import initialize_db
 from timer import Timer
 from http_client import HttpClient
+from sentry_sdk import init as sentry_init
+
+
+settings = get_settings()
 
 timers = []
 connection_manager: ConnectionManager = get_connection_manager()
 
-
 api_description = """
 Swiss AI Center's Core Engine API - The **best** API in the world.
 """
+VERSION = "1.0.0"
+
+if settings.sentry_dsn:
+    sentry_init(
+        dsn=settings.sentry_dsn,
+        release=f"core-engine@{VERSION}",
+        environment=settings.environment.name,
+        traces_sample_rate=1.0,
+        profiles_sample_rate=1.0,
+    )
 
 # Define the FastAPI application with information
 app = FastAPI(
     title="Core Engine API",
     description=api_description,
-    version="1.0.0",
+    version=VERSION,
     contact={
         "name": "Swiss AI Center",
         "url": "https://swiss-ai-center.ch/",
@@ -101,7 +114,6 @@ async def startup_event():
     # Manual instances because startup events doesn't support Dependency Injection
     # https://github.com/tiangolo/fastapi/issues/2057
     # https://github.com/tiangolo/fastapi/issues/425
-    settings = get_settings()
     engine = initialize_db(settings=settings)
     session_generator = get_session(engine)
     session = next(session_generator)