From 157c3b79c49fffb33330415afc65c07ccedf6b5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Wed, 27 Mar 2024 11:32:26 +0100 Subject: [PATCH 1/7] Update for Python 3.11 --- Dockerfile | 2 +- development.Dockerfile | 2 +- docker-compose.yml | 2 +- requirements-all.txt | 94 +++++++++++++++++++++--------------------- requirements.txt | 2 +- 5 files changed, 50 insertions(+), 52 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7591b38..c1057e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base image -FROM python:3.10 +FROM python:3.11 # Install all required packages to run the model RUN apt update && apt install --yes ffmpeg libsm6 libxext6 diff --git a/development.Dockerfile b/development.Dockerfile index 7d8b2dc..45f3b09 100644 --- a/development.Dockerfile +++ b/development.Dockerfile @@ -1,5 +1,5 @@ # Base image -FROM python:3.10 +FROM python:3.11 # Install all required packages to run the model RUN apt update && apt install --yes ffmpeg libsm6 libxext6 diff --git a/docker-compose.yml b/docker-compose.yml index 82a38fe..aaaa2c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: echo '.venv exists and is not empty'; else echo '.venv does not exist or is empty'; - python3.10 -m venv .venv; + python3.11 -m venv .venv; fi && # Activate the virtual environment source .venv/bin/activate && diff --git a/requirements-all.txt b/requirements-all.txt index 63c53a1..8068090 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -1,55 +1,53 @@ -aiobotocore==2.4.1 -aiofile==3.8.7 -aiofiles==22.1.0 -aiohttp==3.8.4 +aiobotocore==2.12.1 +aiofiles==23.2.1 +aiohttp==3.9.3 aioitertools==0.11.0 aiosignal==1.3.1 -anyio==3.7.0 -async-timeout==4.0.2 -attrs==23.1.0 -botocore==1.27.59 -caio==0.9.12 -certifi==2023.5.7 -charset-normalizer==3.1.0 -click==8.1.3 -coverage==7.2.7 -dnspython==2.3.0 -email-validator==2.1.0.post1 -exceptiongroup==1.1.1 -fastapi==0.108.0 -flake8==5.0.4 -frozenlist==1.3.3 +annotated-types==0.6.0 +anyio==4.3.0 +attrs==23.2.0 +botocore==1.34.51 +certifi==2024.2.2 +click==8.1.7 +common_code @ git+https://github.com/swiss-ai-center/common-code.git@53446afc1d1cf0856cfac972d1613dd36691086b +coverage==7.4.4 +dnspython==2.6.1 +email_validator==2.1.1 +fastapi==0.110.0 +flake8==7.0.0 +frozenlist==1.4.1 h11==0.14.0 -httpcore==0.16.3 -httpx==0.23.1 -idna==3.4 +httpcore==1.0.4 +httpx==0.27.0 +idna==3.6 iniconfig==2.0.0 jmespath==1.0.1 -MarkupSafe==2.1.3 +MarkupSafe==2.1.5 mccabe==0.7.0 -multidict==6.0.4 -packaging==23.1 -Pillow==9.1.0 -pluggy==1.2.0 -pycodestyle==2.9.1 -pydantic==2.5.3 -pydantic-settings==2.1.0 -pyflakes==2.5.0 -pytest==7.2.0 -pytest-asyncio==0.20.3 -pytest-cov==4.0.0 -pytest-httpserver==1.0.6 -python-dateutil==2.8.2 -python-dotenv==0.21.0 -PyYAML==6.0 -rfc3986==1.5.0 +multidict==6.0.5 +packaging==24.0 +pillow==10.2.0 +pip==24.0 +pluggy==1.4.0 +pycodestyle==2.11.1 +pydantic==2.6.4 +pydantic-settings==2.2.1 +pydantic_core==2.16.3 +pyflakes==3.2.0 +pytest==8.1.1 +pytest-asyncio==0.23.5.post1 +pytest-cov==4.1.0 +pytest_httpserver==1.0.10 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +PyYAML==6.0.1 +setuptools==65.5.0 six==1.16.0 -sniffio==1.3.0 -starlette==0.29.0 -tomli==2.0.1 -typing_extensions==4.9.0 -urllib3==1.26.16 -uvicorn==0.19.0 -Werkzeug==2.3.6 -wrapt==1.15.0 -yarl==1.9.2 +sniffio==1.3.1 +starlette==0.36.3 +typing_extensions==4.10.0 +urllib3==2.0.7 +uvicorn==0.28.0 +Werkzeug==3.0.1 +wrapt==1.16.0 +yarl==1.9.4 diff --git a/requirements.txt b/requirements.txt index 1907b96..89737bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ common-code[test] @ git+https://github.com/swiss-ai-center/common-code.git@main -Pillow==9.1.0 +pillow==10.2.0 From 5595ea416818c8c4484894bd6872fae785deba7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Wed, 27 Mar 2024 11:32:40 +0100 Subject: [PATCH 2/7] Remove common-code from requirements-all.txt --- requirements-all.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 8068090..d27d9e0 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -9,7 +9,6 @@ attrs==23.2.0 botocore==1.34.51 certifi==2024.2.2 click==8.1.7 -common_code @ git+https://github.com/swiss-ai-center/common-code.git@53446afc1d1cf0856cfac972d1613dd36691086b coverage==7.4.4 dnspython==2.6.1 email_validator==2.1.1 From 76a18702a66533afad728e3fae146f8a444ea838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Wed, 27 Mar 2024 11:33:45 +0100 Subject: [PATCH 3/7] Add docker gateway for Linux local dev --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index aaaa2c4..eecd2d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,8 @@ services: environment: - ENGINE_URLS=["http://host.docker.internal:8080"] - SERVICE_URL=http://host.docker.internal:${SERVICE_PORT} + extra_hosts: + - "host.docker.internal:host-gateway" ports: - ${SERVICE_PORT}:9090 working_dir: /workspaces/service From 1da1b8facb92adddef5e5018884cfc1fee06de26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Wed, 27 Mar 2024 11:35:15 +0100 Subject: [PATCH 4/7] Update actions in workflow --- .github/workflows/workflow.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 57bbb46..bbdc952 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -65,7 +65,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Lint Python app uses: swiss-ai-center/common-code/.github/actions/lint-python-app@main @@ -78,7 +78,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Test Python app uses: swiss-ai-center/common-code/.github/actions/test-python-app@main @@ -92,7 +92,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' && success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod') && (vars.DEPLOY_DEV == 'true' || vars.DEPLOY_PROD == 'true') }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build and push Docker image to GitHub id: build-and-push-docker-image-to-github @@ -111,7 +111,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/main' && vars.DEPLOY_DEV == 'true' }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get service Docker image SHA tag shell: bash @@ -153,7 +153,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/prod' && vars.DEPLOY_PROD == 'true' }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get service Docker image SHA tag shell: bash From 21c22704179a6062c5743d8c13fa8cd57f8f163f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Wed, 27 Mar 2024 11:37:20 +0100 Subject: [PATCH 5/7] Update ingress and workflow for production --- .github/workflows/workflow.yml | 9 +++++++-- kubernetes/ingress.yml | 12 +++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index bbdc952..c5c91f8 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -89,7 +89,7 @@ jobs: release: needs: test runs-on: ubuntu-latest - if: ${{ vars.RUN_CICD == 'true' && success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod') && (vars.DEPLOY_DEV == 'true' || vars.DEPLOY_PROD == 'true') }} + if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/main' && (vars.DEPLOY_DEV == 'true' || vars.DEPLOY_PROD == 'true') }} steps: - name: Clone repository uses: actions/checkout@v4 @@ -134,6 +134,11 @@ jobs: engine-announce-retries: ${{ env.DEV_ENGINE_ANNOUNCE_RETRIES }} engine-announce-retry-delay: ${{ env.DEV_ENGINE_ANNOUNCE_RETRY_DELAY }} + - name: Remove unnecessary keys from configuration files + uses: swiss-ai-center/common-code/.github/actions/remove-unnecessary-keys-from-kubernetes-configuration-files@main + with: + configuration-files-location: ./kubernetes + - name: Deploy service on the Kubernetes cluster uses: swiss-ai-center/common-code/.github/actions/execute-command-on-kubernetes-cluster@main with: @@ -150,7 +155,7 @@ jobs: deploy-prod: needs: release runs-on: ubuntu-latest - if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/prod' && vars.DEPLOY_PROD == 'true' }} + if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/main' && vars.DEPLOY_PROD == 'true' }} steps: - name: Clone repository uses: actions/checkout@v4 diff --git a/kubernetes/ingress.yml b/kubernetes/ingress.yml index fb2020a..22ce83a 100644 --- a/kubernetes/ingress.yml +++ b/kubernetes/ingress.yml @@ -3,14 +3,15 @@ kind: Ingress metadata: name: my-service-ingress annotations: + cert-manager.io/issuer: "letsencrypt" nginx.ingress.kubernetes.io/proxy-body-size: "16m" nginx.org/client-max-body-size: "16m" spec: rules: - - host: my-service.example.com - http: - paths: - - path: / + - host: my-service.example.com + http: + paths: + - path: / pathType: Prefix backend: service: @@ -19,4 +20,5 @@ spec: number: 80 tls: - hosts: - - my-service.example.com + - my-service.example.com + secretName: my-service-example-com-tls-cert From 233a7c2121f6a895b1d8e6802d04a47880c5938c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Thu, 11 Apr 2024 08:32:51 +0200 Subject: [PATCH 6/7] Add missing config in ingress --- kubernetes/ingress.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/ingress.yml b/kubernetes/ingress.yml index 22ce83a..18dc546 100644 --- a/kubernetes/ingress.yml +++ b/kubernetes/ingress.yml @@ -7,6 +7,7 @@ metadata: nginx.ingress.kubernetes.io/proxy-body-size: "16m" nginx.org/client-max-body-size: "16m" spec: + ingressClassName: nginx rules: - host: my-service.example.com http: From c3576b3a9858a2a6f48568d42c1978e587624576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Thu, 11 Apr 2024 10:10:53 +0200 Subject: [PATCH 7/7] Fix ingress indent --- kubernetes/ingress.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/ingress.yml b/kubernetes/ingress.yml index 18dc546..19aba71 100644 --- a/kubernetes/ingress.yml +++ b/kubernetes/ingress.yml @@ -13,12 +13,12 @@ spec: http: paths: - path: / - pathType: Prefix - backend: - service: - name: my-service-service - port: - number: 80 + pathType: Prefix + backend: + service: + name: my-service-service + port: + number: 80 tls: - hosts: - my-service.example.com