From 1ca24a26742e52f11def02dbfb9369e44dc6054c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Tue, 26 Mar 2024 15:57:16 +0100 Subject: [PATCH 1/9] Update for Python 3.11 --- Dockerfile | 2 +- README.md | 2 +- development.Dockerfile | 2 +- docker-compose.yml | 2 +- requirements-all.txt | 117 ++++++++++++++++++++--------------------- requirements.txt | 2 +- 6 files changed, 62 insertions(+), 65 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b0d804..e11c0c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base image -FROM python:3.10 +FROM python:3.11 # Install all required packages to run the model # TODO: 1. Add any additional packages required to run your model diff --git a/README.md b/README.md index d86ce9b..148e202 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ some categories and as there is simply a larger cardinality in the number of cla 1. Create and activate the virtual environment: ```sh -python3.10 -m venv .venv +python3.11 -m venv .venv source .venv/bin/activate ``` diff --git a/development.Dockerfile b/development.Dockerfile index ad25d10..343f296 100644 --- a/development.Dockerfile +++ b/development.Dockerfile @@ -1,2 +1,2 @@ # Base image -FROM python:3.10 +FROM python:3.11 diff --git a/docker-compose.yml b/docker-compose.yml index 82a38fe..bd6ac6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: echo '.venv exists and is not empty'; else echo '.venv does not exist or is empty'; - python3.10 -m venv .venv; + python3.11 -m venv .venv; fi && # Activate the virtual environment source .venv/bin/activate && diff --git a/requirements-all.txt b/requirements-all.txt index fd070a2..c0b78ee 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -1,88 +1,85 @@ absl-py==2.1.0 -aiobotocore==2.4.1 -aiofiles==22.1.0 -aiohttp==3.9.1 +aiobotocore==2.12.1 +aiofiles==23.2.1 +aiohttp==3.9.3 aioitertools==0.11.0 aiosignal==1.3.1 annotated-types==0.6.0 -anyio==4.2.0 +anyio==4.3.0 astunparse==1.6.3 -async-timeout==4.0.3 attrs==23.2.0 -botocore==1.27.59 -cachetools==5.3.2 -certifi==2023.11.17 +botocore==1.34.51 +certifi==2024.2.2 charset-normalizer==3.3.2 click==8.1.7 -coverage==7.4.0 -dnspython==2.5.0 -email-validator==2.1.0.post1 -exceptiongroup==1.2.0 -fastapi==0.108.0 -flake8==5.0.4 -flatbuffers==23.5.26 +common_code @ git+https://github.com/swiss-ai-center/common-code.git@53446afc1d1cf0856cfac972d1613dd36691086b +coverage==7.4.4 +dnspython==2.6.1 +email_validator==2.1.1 +fastapi==0.110.0 +flake8==7.0.0 +flatbuffers==24.3.25 frozenlist==1.4.1 gast==0.5.4 -google-auth==2.26.2 -google-auth-oauthlib==1.2.0 google-pasta==0.2.0 -grpcio==1.60.0 +grpcio==1.62.1 h11==0.14.0 h5py==3.10.0 -httpcore==0.16.3 -httpx==0.23.1 +httpcore==1.0.4 +httpx==0.27.0 idna==3.6 ImageHash==4.3.1 iniconfig==2.0.0 jmespath==1.0.1 -keras==2.15.0 -libclang==16.0.6 -Markdown==3.5.2 -MarkupSafe==2.1.4 +keras==3.1.1 +libclang==18.1.1 +Markdown==3.6 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 mccabe==0.7.0 -ml-dtypes==0.2.0 -multidict==6.0.4 -numpy==1.26.3 -oauthlib==3.2.2 +mdurl==0.1.2 +ml-dtypes==0.3.2 +multidict==6.0.5 +namex==0.0.7 +numpy==1.26.4 opt-einsum==3.3.0 -packaging==23.2 +optree==0.11.0 +packaging==24.0 pillow==10.2.0 +pip==24.0 pip-chill==1.0.3 -pluggy==1.3.0 -protobuf==4.23.4 -pyasn1==0.5.1 -pyasn1-modules==0.3.0 -pycodestyle==2.9.1 -pydantic==2.5.3 -pydantic-settings==2.1.0 -pydantic_core==2.14.6 -pyflakes==2.5.0 -pytest==7.2.0 -pytest-asyncio==0.20.3 -pytest-cov==4.0.0 -pytest-httpserver==1.0.6 -python-dateutil==2.8.2 -python-dotenv==0.21.0 +pluggy==1.4.0 +protobuf==4.25.3 +pycodestyle==2.11.1 +pydantic==2.6.4 +pydantic-settings==2.2.1 +pydantic_core==2.16.3 +pyflakes==3.2.0 +Pygments==2.17.2 +pytest==8.1.1 +pytest-asyncio==0.23.5.post1 +pytest-cov==4.1.0 +pytest_httpserver==1.0.10 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 PyWavelets==1.5.0 -PyYAML==6.0 +PyYAML==6.0.1 requests==2.31.0 -requests-oauthlib==1.3.1 -rfc3986==1.5.0 -rsa==4.9 +rich==13.7.1 scipy==1.12.0 +setuptools==65.5.0 six==1.16.0 -sniffio==1.3.0 -starlette==0.29.0 -tensorboard==2.15.1 +sniffio==1.3.1 +starlette==0.36.3 +tensorboard==2.16.2 tensorboard-data-server==0.7.2 -tensorflow==2.15.0 -tensorflow-estimator==2.15.0 -tensorflow-io-gcs-filesystem==0.34.0 +tensorflow==2.16.1 +tensorflow-io-gcs-filesystem==0.36.0 termcolor==2.4.0 -tomli==2.0.1 -typing_extensions==4.9.0 -urllib3==1.26.18 -uvicorn==0.19.0 -Werkzeug==2.3.6 -wrapt==1.14.1 +typing_extensions==4.10.0 +urllib3==2.0.7 +uvicorn==0.28.0 +Werkzeug==3.0.1 +wheel==0.43.0 +wrapt==1.16.0 yarl==1.9.4 diff --git a/requirements.txt b/requirements.txt index d6c4a47..8b24449 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ common-code[test] @ git+https://github.com/swiss-ai-center/common-code.git@main imagehash==4.3.1 pip-chill==1.0.3 -tensorflow==2.15.0 +tensorflow==2.16.1 From 7b43cf590336d9f63b4d7e66282bfe66383d39e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Tue, 26 Mar 2024 15:57:35 +0100 Subject: [PATCH 2/9] Remove common-code from requirements-all.txt --- requirements-all.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index c0b78ee..783d85b 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -12,7 +12,6 @@ botocore==1.34.51 certifi==2024.2.2 charset-normalizer==3.3.2 click==8.1.7 -common_code @ git+https://github.com/swiss-ai-center/common-code.git@53446afc1d1cf0856cfac972d1613dd36691086b coverage==7.4.4 dnspython==2.6.1 email_validator==2.1.1 From 037fa405760ccad47b4df20658b99ef3eee39b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Tue, 26 Mar 2024 15:59:05 +0100 Subject: [PATCH 3/9] Add docker gateway for Linux local dev --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index bd6ac6e..3f54799 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,8 @@ services: environment: - ENGINE_URLS=["http://host.docker.internal:8080"] - SERVICE_URL=http://host.docker.internal:${SERVICE_PORT} + extra_hosts: + - "host.docker.internal:host-gateway" ports: - ${SERVICE_PORT}:9090 working_dir: /workspaces/service From 93536cc8fdbd3044e5778c878c84a25b7bdfa903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Tue, 26 Mar 2024 15:59:52 +0100 Subject: [PATCH 4/9] Update actions in workflow --- .github/workflows/workflow.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 57bbb46..bbdc952 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -65,7 +65,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Lint Python app uses: swiss-ai-center/common-code/.github/actions/lint-python-app@main @@ -78,7 +78,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Test Python app uses: swiss-ai-center/common-code/.github/actions/test-python-app@main @@ -92,7 +92,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' && success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod') && (vars.DEPLOY_DEV == 'true' || vars.DEPLOY_PROD == 'true') }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build and push Docker image to GitHub id: build-and-push-docker-image-to-github @@ -111,7 +111,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/main' && vars.DEPLOY_DEV == 'true' }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get service Docker image SHA tag shell: bash @@ -153,7 +153,7 @@ jobs: if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/prod' && vars.DEPLOY_PROD == 'true' }} steps: - name: Clone repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get service Docker image SHA tag shell: bash From b7ad221ccb68babfa18866907668d67979fa3402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Tue, 26 Mar 2024 16:03:14 +0100 Subject: [PATCH 5/9] Update ingress and workflow for production --- .github/workflows/workflow.yml | 9 +++++++-- kubernetes/ingress.yml | 12 +++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index bbdc952..8b84016 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -89,7 +89,7 @@ jobs: release: needs: test runs-on: ubuntu-latest - if: ${{ vars.RUN_CICD == 'true' && success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod') && (vars.DEPLOY_DEV == 'true' || vars.DEPLOY_PROD == 'true') }} + if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/main' && (vars.DEPLOY_DEV == 'true' || vars.DEPLOY_PROD == 'true') }} steps: - name: Clone repository uses: actions/checkout@v4 @@ -134,6 +134,11 @@ jobs: engine-announce-retries: ${{ env.DEV_ENGINE_ANNOUNCE_RETRIES }} engine-announce-retry-delay: ${{ env.DEV_ENGINE_ANNOUNCE_RETRY_DELAY }} + - name: Remove unnecessary keys from configuration files + uses: swiss-ai-center/common-code/.github/actions/remove-unnecessary-keys-from-kubernetes-configuration-files@main + with: + configuration-files-location: ./kubernetes + - name: Deploy service on the Kubernetes cluster uses: swiss-ai-center/common-code/.github/actions/execute-command-on-kubernetes-cluster@main with: @@ -150,7 +155,7 @@ jobs: deploy-prod: needs: release runs-on: ubuntu-latest - if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/prod' && vars.DEPLOY_PROD == 'true' }} + if: ${{ vars.RUN_CICD == 'true' && success() && github.ref == 'refs/heads/main' && vars.DEPLOY_PROD == 'true' }} steps: - name: Clone repository uses: actions/checkout@v4 diff --git a/kubernetes/ingress.yml b/kubernetes/ingress.yml index fb2020a..23c8f2d 100644 --- a/kubernetes/ingress.yml +++ b/kubernetes/ingress.yml @@ -3,14 +3,15 @@ kind: Ingress metadata: name: my-service-ingress annotations: + cert-manager.io/issuer: "letsencrypt" nginx.ingress.kubernetes.io/proxy-body-size: "16m" nginx.org/client-max-body-size: "16m" spec: rules: - - host: my-service.example.com - http: - paths: - - path: / + - host: my-service.example.com + http: + paths: + - path: / pathType: Prefix backend: service: @@ -19,4 +20,5 @@ spec: number: 80 tls: - hosts: - - my-service.example.com + - my-service.example.com + secretName: my-service-example-com-tls-cert From 1d437c487b46e8eb649d5aeba4a52b434f962ae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Thu, 11 Apr 2024 08:30:34 +0200 Subject: [PATCH 6/9] Add missing config in ingress --- kubernetes/ingress.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/ingress.yml b/kubernetes/ingress.yml index 23c8f2d..0778bbd 100644 --- a/kubernetes/ingress.yml +++ b/kubernetes/ingress.yml @@ -7,6 +7,7 @@ metadata: nginx.ingress.kubernetes.io/proxy-body-size: "16m" nginx.org/client-max-body-size: "16m" spec: + ingressClassName: nginx rules: - host: my-service.example.com http: From 3e00803d07dbe04d0b2282ffa1aafb79e3dd7025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Thu, 11 Apr 2024 08:55:00 +0200 Subject: [PATCH 7/9] Fix indentation in workflow file --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 8b84016..c5c91f8 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -134,7 +134,7 @@ jobs: engine-announce-retries: ${{ env.DEV_ENGINE_ANNOUNCE_RETRIES }} engine-announce-retry-delay: ${{ env.DEV_ENGINE_ANNOUNCE_RETRY_DELAY }} - - name: Remove unnecessary keys from configuration files + - name: Remove unnecessary keys from configuration files uses: swiss-ai-center/common-code/.github/actions/remove-unnecessary-keys-from-kubernetes-configuration-files@main with: configuration-files-location: ./kubernetes From c1a08c105cd21fe557143724456a3b76f733ea93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Thu, 11 Apr 2024 10:05:58 +0200 Subject: [PATCH 8/9] Fix ingress indent --- kubernetes/ingress.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/ingress.yml b/kubernetes/ingress.yml index 0778bbd..6b5b103 100644 --- a/kubernetes/ingress.yml +++ b/kubernetes/ingress.yml @@ -13,12 +13,12 @@ spec: http: paths: - path: / - pathType: Prefix - backend: - service: - name: my-service-service - port: - number: 80 + pathType: Prefix + backend: + service: + name: my-service-service + port: + number: 80 tls: - hosts: - my-service.example.com From 4e8d92715c3ad499f51e4b26ae59e2f2e9b2f80e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Marquis?= Date: Fri, 12 Apr 2024 12:30:37 +0200 Subject: [PATCH 9/9] Adjust test condition to look only for engine related warning --- tests/test_service.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_service.py b/tests/test_service.py index b280131..bab1ac8 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -64,7 +64,7 @@ def unreachable_engine_instance_fixture(httpserver: HTTPServer): def app_with_reachable_engine_instance(reachable_engine_instance: HTTPServer): def get_settings_override(): settings = get_settings() - settings.engine_urls = reachable_engine_instance.url_for("") + settings.engine_urls = [reachable_engine_instance.url_for("")] settings.engine_announce_retries = 2 settings.engine_announce_retry_delay = 1 settings.max_tasks = 2 @@ -83,7 +83,7 @@ def get_settings_override(): def app_with_unreachable_engine_instance(unreachable_engine_instance: HTTPServer): def get_settings_override(): settings = get_settings() - settings.engine_urls = unreachable_engine_instance.url_for("") + settings.engine_urls = [unreachable_engine_instance.url_for("")] settings.engine_announce_retries = 2 settings.engine_announce_retry_delay = 1 settings.max_tasks = 2 @@ -111,10 +111,11 @@ def test_announce_to_reachable_engine( # This is not a good way to test the app as any other warnings will make the test # passes. warning_logs_found = False - # for record in caplog.records: - # if record.levelname == "WARNING": - # warning_logs_found = True - # break + for record in caplog.records: + if record.levelname == "WARNING": + if "Failed to notify the engine" in record.message: + warning_logs_found = True + break assert not warning_logs_found @@ -134,8 +135,9 @@ def test_announce_to_unreachable_engine( warning_logs_found = False for record in caplog.records: if record.levelname == "WARNING": - warning_logs_found = True - break + if "Failed to notify the engine" in record.message: + warning_logs_found = True + break assert warning_logs_found