Update dev tools config (#929)

Use poetry and update docker configurations

.dockerignore

@@ -0,0 +1,7 @@
+# this file is only used to build the frontend image
+# NOT for the python api (see api/.dockerignore for that)
+common/node_modules/**
+frontend/node_modules/**
+alerting/*
+api/*
+docs/*

.github/workflows/api.yml

@@ -0,0 +1,56 @@
+# for comment on PR, follow instruction for: https://github.com/marketplace/actions/sticky-pull-request-comment
+name: PRISM CI - API
+on:
+  # Run on all pull requests and on pushes to master.
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  api_lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+      # run the matrix jobs one after the other, so they can benefit from caching
+      max-parallel: 1
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          # Version of Poetry to use
+          version: 1.6.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+      - name: Install dependencies
+        run: |
+          cd api
+          poetry install
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      - name: Check formatting with black and isort
+        run: |
+          cd api
+          poetry run black --check .
+          poetry run isort --check .
+
+  api_build:
+    name: build and test api
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Install Docker Compose
+        run: |
+          DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
+          mkdir -p $DOCKER_CONFIG/cli-plugins
+          curl -SL https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-`uname -s`-`uname -m` -o $DOCKER_CONFIG/cli-plugins/docker-compose
+          chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose
+          docker compose version
+      - name: Build containers
+        run: cd api && make build
+      - name: Run tests
+        run: cd api && make api-test
+

.github/workflows/build.yml → .github/workflows/frontend.yml

@@ -1,5 +1,5 @@
 # for comment on PR, follow instruction for: https://github.com/marketplace/actions/sticky-pull-request-comment
-name: PRISM CI
+name: PRISM CI - Frontend
 on:
   # Run on all pull requests and on pushes to master.
   pull_request:
@@ -172,47 +172,3 @@ jobs:
           project: "frontend/build/."
           login: ${{ secrets.surge_login }}
           token: ${{ secrets.surge_token }}
-
-  api_lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.10"]
-      # run the matrix jobs one after the other, so they can benefit from caching
-      max-parallel: 1
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-        with:
-          # Version of Poetry to use
-          version: 1.4.2
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-      - name: Install dependencies
-        run: |
-          cd api
-          poetry install
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-      - name: Check formatting with black and isort
-        run: |
-          cd api
-          poetry run black --check .
-          poetry run isort --check .
-
-  api_build:
-    name: build api
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Install Docker Compose
-        run: |
-          curl -L https://github.com/docker/compose/releases/download/1.11.2/docker-compose-`uname -s`-`uname -m` > ~/docker-compose
-          chmod +x ~/docker-compose
-          sudo mv ~/docker-compose /usr/local/bin/docker-compose
-      - name: Build containers
-        run: cd api && docker-compose build api
-      - name: Run tests
-        run: cd api && make api-test

README.md

@@ -284,6 +284,18 @@ This project was bootstrapped with [Create React App](https://github.com/faceboo
 - **Mapping** Uses [MapLibre](https://maplibre.org/maplibre-gl-js-docs/api/). The app supports Maptiler and Mapbox styles. To use Mapbox styles, you will need to create a token and add it as `REACT_APP_MAPBOX_TOKEN` in a `.env` file at the root folder. Then specify your style url using `REACT_APP_DEFAULT_STYLE`.
 - **WFP authentication** Uses [msal](https://github.com/AzureAD/microsoft-authentication-library-for-js). You need to include within your .env file the variables `REACT_APP_OAUTH_CLIENT_ID`, `REACT_APP_OAUTH_AUTHORITY` and `REACT_APP_OAUTH_REDIRECT_URI`. Also, set the `WFPAuthRequired` flag within the country prism.json file
 
+### Developing the frontend
+
+The following commands should get you a local development instance of the frontend:
+
+```
+cd frontend
+yarn clean
+yarn install
+yarn setup:common
+REACT_APP_COUNTRY=cambodia yarn start
+```
+
 ### Available Scripts
 
 In the project directory, you can run:

alerting/package.json

@@ -22,7 +22,7 @@
     "migration:revert": "yarn typeorm migration:revert",
     "typeorm-seeding": "ts-node $(yarn bin typeorm-seeding)",
     "alert-worker": "yarn ts-node src/alert-worker.ts",
-    "docker-alert": "docker-compose run --entrypoint 'yarn alert-worker' alerting-node"
+    "docker-alert": "docker compose run --entrypoint 'yarn alert-worker' alerting-node"
   },
   "dependencies": {
     "@turf/bbox": "^6.3.0",

api/Dockerfile

@@ -1,21 +1,51 @@
-# only image with python 3.10 on it
-# other GDAL images use python 3.8
-# This provides GDAL 3.6dev
-FROM osgeo/gdal:ubuntu-small-latest
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.7.0
 
-COPY ./app /app
+ENV PYTHONPATH=/
 
-WORKDIR /app/
+ENV POETRY_VERSION=1.6.1
+# install poetry in this location
+ENV POETRY_HOME="/opt/poetry"
 
-ENV PYTHONPATH=/
+# do not ask any interactive question
+ENV POETRY_NO_INTERACTION=1
+# set the cache location so we can mount docker buildx cache on it
+ENV POETRY_CACHE_DIR=/root/.cache
+
+# set poetry to use the system python env so we get all
+# the GDAL packages which are already installed. This goes
+# against best practice, but follows the logic in place before
+# the switch to poetry
+ENV POETRY_VIRTUALENVS_CREATE=false
+
+ENV PATH="$POETRY_HOME/bin:$PATH"
+
+RUN --mount=type=cache,target=/root/.cache curl -sSL https://install.python-poetry.org | python3 -
 
-COPY requirements.txt .
+# COPY requirements.txt .
+# copy pyproject so that black and isort use the right config
+COPY pyproject.toml /
 
-RUN apt update
-RUN apt install -y python3-pip libpq-dev
+COPY poetry.lock /
+
+RUN --mount=type=cache,target=/var/cache/apt \
+    apt-get update && \
+    apt-get --no-install-recommends install -y python3-pip libpq-dev curl python3-dev build-essential
 RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
-RUN playwright install chromium
-RUN playwright install-deps
+
+RUN --mount=type=cache,target=/root/.cache \
+    poetry install
+
+# RUN --mount=type=cache,target=/root/.cache poetry run playwright install chromium
+RUN poetry run playwright install chromium
+# playwright installs a lot of apt packages
+RUN --mount=type=cache,target=/root/.cache \
+    --mount=type=cache,target=/var/lib/apt \
+    --mount=type=cache,target=/var/cache/apt \
+    poetry run playwright install-deps
+
+# keep this copy towards the end to help with caching previous layers
+COPY ./app /app
+
+WORKDIR /app/
 
 EXPOSE 80

api/Makefile

@@ -1,34 +1,37 @@
 build:
-	docker compose build api
+	docker compose -f docker-compose.yml -f docker-compose.test.yml build
+	docker compose -f docker-compose.yml -f docker-compose.test.yml pull
 
 api:
-	docker-compose -f docker-compose.yml -f docker-compose.develop.yml up
+	docker compose -f docker-compose.yml -f docker-compose.develop.yml up
 
 deploy:
-	docker-compose build && docker-compose -f docker-compose.yml -f docker-compose.deploy.yml up -d
+	docker compose build && docker-compose -f docker-compose.yml -f docker-compose.deploy.yml up -d
 
 logs:
-	docker-compose -f docker-compose.yml -f docker-compose.deploy.yml logs -f
+	docker compose -f docker-compose.yml -f docker-compose.deploy.yml logs -f
 
 api-lint:
-	docker-compose run --no-deps api bash -c "black ."
-	docker-compose run --no-deps api bash -c "isort ."
+	docker compose run --no-deps api bash -c "poetry run black ."
+	docker compose run --no-deps api bash -c "poetry run isort ."
 
 api-typecheck:
-	docker-compose run --no-deps api bash -c "mypy ."
+	docker compose run --no-deps api bash -c "mypy ."
 
 test-services:
-	docker-compose -f docker-compose.yml -f docker-compose.test.yml up -d
+	docker compose -f docker-compose.yml -f docker-compose.test.yml up -d
 
 api-test: test-services
-	docker-compose -f docker-compose.yml -f docker-compose.test.yml run api bash -c "pytest -s tests"
+	docker compose -f docker-compose.yml -f docker-compose.test.yml run api bash -c "../wait-for-it.sh frontend:3000 -t 180"
+	docker compose -f docker-compose.yml -f docker-compose.test.yml run api bash -c "pytest -s tests"
+	docker compose -f docker-compose.yml -f docker-compose.test.yml down
 
 api-add-users:
-	docker-compose -f docker-compose.yml -f docker-compose.develop.yml run api bash -c "python scripts/add_users.py"
+	docker compose -f docker-compose.yml -f docker-compose.develop.yml run api bash -c "python scripts/add_users.py"
 
 BACKEND_COVERAGE=pytest --cov=app --cov-config .coveragerc --cov-fail-under=30 --cov-report term-missing
 api-coverage:
-	docker-compose run --no-deps api ${BACKEND_COVERAGE}
+	docker compose run --no-deps api ${BACKEND_COVERAGE}
 
 # Run tests for all components.
 test:
@@ -37,7 +40,7 @@ test:
 	$(MAKE) api-coverage
 
 bash:
-	docker-compose -f docker-compose.yml -f docker-compose.develop.yml run api bash
+	docker compose -f docker-compose.yml -f docker-compose.develop.yml run api bash
 
 # [Dummy dependency to force a make command to always run.]
 FORCE:

api/README.md

@@ -73,7 +73,6 @@ Returns armed conflict incidents using ACLED api. Make sure to have the defined
 - `?fields`, Comma separated string which specifies the fields to be returned per incident.
 - `?event_date`, Return incidents only matching the given value with format YYYY-MM-DD
 
-
 ### `/kobo/forms` (GET)
 
 Returns all form responses using Kobo API
@@ -112,7 +111,7 @@ make api
 To run flask api together with database within same network, run:
 
 ```
-docker-compose -f ./docker-compose.develop.yml -f ../alerting/docker-compose.yml up
+docker compose -f ./docker-compose.develop.yml -f ../alerting/docker-compose.yml up
 ```
 
 ### Tests
@@ -123,6 +122,22 @@ To run linting and tests, run:
 make test
 ```
 
+#### Debugging playwright tests
+
+To run python tests outside of docker, run "make localtests". This will set them up to run outside docker, so that
+playwright can run in debug mode, with a visible browser.
+
+- start the frontend in docker with `make test-services`.
+- in a shell (tested in bash only):
+
+```bash
+cd api/app
+mv tests/conftest.py-template tests/conftest.py
+KOBO_USERNAME=ovio KOBO_PW=pwd PWDEBUG=1 poetry run pytest -s tests -k test_download_report
+```
+
+This should open playwright in debug mode, with a browser window and a debugging one. More info: https://playwright.dev/python/docs/debug
+
 ## Deployments
 
 We are using [docs.traefik.io](https://docs.traefik.io/)

api/app/auth.py

@@ -3,6 +3,7 @@
 import hashlib
 import logging
 import secrets
+from typing import Annotated
 
 from app.database.database import AuthDataBase
 from app.database.user_info_model import UserInfoModel
@@ -18,6 +19,11 @@
 auth_db = AuthDataBase()
 
 if not auth_db.active:
+    # in local development, this condition might be true
+    # in which case the openapi introspection can generate
+    # strange results in the openapi.json output.
+    # eg. /kobo/forms might require a body in a GET request
+    # which is invalid.
     depends = lambda *_: True
 
 
@@ -36,7 +42,9 @@ def verify_hash(password: str, saved_salt: str) -> bytes:
     return key
 
 
-def validate_user(credentials: HTTPBasicCredentials = depends) -> UserInfoModel:
+def validate_user(
+    credentials: Annotated[HTTPBasicCredentials, depends]
+) -> UserInfoModel:
     """Validate user info."""
     if not auth_db.active:
         return UserInfoModel(access={})