diff --git a/.ci.env b/.ci.env new file mode 100644 index 0000000..9648744 --- /dev/null +++ b/.ci.env @@ -0,0 +1,22 @@ +# This file is used to allow CI to start the compose services. It will typcially +# not need to be modified. + +MYSQL_DATABASE=mysql_database +MYSQL_USER=mysql_user +MYSQL_PASSWORD=mysql_password +MYSQL_PORT=3306 +MYSQL_ROOT_PASSWORD=test123! +MYSQL_HOST=db + +ALLOWED_HOSTS='127.0.0.1,localhost' +CORS_ORIGIN_WHITELIST='http://127.0.0.1:3000,http://localhost:3000' +CSRF_TRUSTED_ORIGINS='http://127.0.0.1:8000,http://localhost:8000' + +SECRET_KEY='secret_key' +DEBUG='True' + +# Add the following to your local .env file. They will be used in the CI process +# and you can largely forget about them, but including them in your .env file +# will act like a safe default and help suppress warnings. +REGISTRY="" +TAG="" diff --git a/.env.example b/.env.example index 9b824fe..17dfca2 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,9 @@ CSRF_TRUSTED_ORIGINS='http://127.0.0.1:8000,http://localhost:8000' SECRET_KEY='secret_key' DEBUG='True' + +# Add the following to your local .env file. They will be used in the CI process +# and you can largely forget about them, but including them in your .env file +# will act like a safe default and help suppress warnings. +REGISTRY="" +TAG="" diff --git a/.github/workflows/build-and-deploy.yaml b/.github/workflows/build-and-deploy.yaml new file mode 100644 index 0000000..9f706dc --- /dev/null +++ b/.github/workflows/build-and-deploy.yaml @@ -0,0 +1,57 @@ +name: 'Build and deploy application containers' +on: + push: +jobs: + build-tag-push-deploy: + runs-on: ubuntu-latest + # CI/CD will run on these branches + if: > + github.ref == 'refs/heads/master' || + github.ref == 'refs/heads/development' + strategy: + matrix: + # Specify the docker-compose services to build images from + service: [sdwebapp, sdnginx] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Login to GitHub container registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: cmu-delphi-deploy-machine + password: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }} + - name: Create container image tags + id: image-tag + run: | + baseRef="${GITHUB_REF#*/}" + baseRef="${baseRef#*/}" + case "${baseRef}" in + master) + image_tag="latest" + ;; + *) + image_tag="${baseRef//\//_}" # replace `/` with `_` in branch name + ;; + esac + echo "IMAGE_TAG=${image_tag}" >> $GITHUB_OUTPUT + - name: Copy env file + run: | + cp ./.ci.env ./.env + - name: Set up docker-compose + uses: ndeloof/install-compose-action@v0.0.1 + - name: docker-compose build --push + run: | + docker-compose build --push ${{ matrix.service }} + env: + TAG: ":${{ steps.image-tag.outputs.IMAGE_TAG }}" + REGISTRY: "ghcr.io/${{ github.repository_owner }}/" + - name: docker-compose down + run: | + docker-compose down + - name: Trigger smee.io webhook to pull new container images + run: | + curl -H "Authorization: Bearer ${{ secrets.DELPHI_DEPLOY_WEBHOOK_TOKEN }}" \ + -X POST ${{ secrets.DELPHI_DEPLOY_WEBHOOK_URL }} \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "repository=ghcr.io/${{ github.repository }}-${{ matrix.service }}&tag=${{ steps.image-tag.outputs.IMAGE_TAG }}" diff --git a/Dockerfile b/Dockerfile index 3a3f826..b53384f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,11 +12,11 @@ RUN apt-get install python3 -y RUN apt-get install python3-pip -y RUN python3 -m pip install --upgrade pip RUN pip3 install pipenv +RUN pipenv lock RUN pipenv requirements > requirements.txt RUN pip3 install -r requirements.txt WORKDIR /home/python -# RUN pipenv install --system --deploy COPY /src . COPY /gunicorn/gunicorn.py . ENV PATH="/home/python/.local/bin:${PATH}" diff --git a/Pipfile b/Pipfile index b742f39..719f53d 100644 --- a/Pipfile +++ b/Pipfile @@ -27,6 +27,7 @@ django-coverage-plugin = "*" django-extensions-models = "*" mypy = "*" django-stubs = "*" +tzdata = "*" [dev-packages] flake8 = "*" diff --git a/README.md b/README.md index a19b7a0..f92ab25 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,27 @@ $ docker-compose up Open `http://localhost:8000` to view it in the browser +### To run via docker and emulate production + +Though probably not necessary in most cases, if you want to test/modify/emulate how this will run in production you can: + +- In `.env` set: +``` +DEBUG = 'False' +``` +- Modify the app container's command in `docker-compose.yaml` to run: +``` +gunicorn signal_documentation.wsgi:application --bind 0.0.0.0:8000" + +*(Essentially you'll replace just the last line of the command, switching out the "runserver" line) +``` + +Open `http://localhost` to view it in the browser. In this usage your request will be serviced by Nginx instead of the application directly. + +The primary use case for this will be when making changes to the Nginx container image that runs in production and hosts the static file content, or also if making changes to the Gunicorn config. + +Changes of this sort should be carefully evaluated as they may require interaction with systems managed by devops folks. + ## [Django admin](https://docs.djangoproject.com/en/4.1/ref/contrib/admin/) web interface (user should be `is_staff` or `is_superuser`) `http://localhost:8000/admin` @@ -126,3 +147,22 @@ Othervice you will receive Errors during import process: 3. Import `Signal.base` fields with `SignalBaseResource` - [http://localhost:8000/admin/signals/signal/import/](http://localhost:8000/admin/signals/signal/import/) ![Import `Signal.base` field](./docs/image-5.png) ![Confirm importing `Signal.base` fields](./docs/image-6.png) + +## Deployment + +This application gets deployed (at a minimum) to two environmetns: + +Production - + +Staging - + +Each environment is essentially a bunch of different services all governed by `docker-compose`, running across multiple hosts, with various layering of proxies and load balancers. + +### Basic workflow + +- A PR merged to either `development` or `master` will trigger CI to build container images that are then tagged (based on the branch name and ":latest" respectively) and stored in our GitHub Packages container image repository. +- CI triggers a webhook that tells the host systems to pull and run new container images and restart any services that have been updated. + +### Control of the deployed environment + +The environment and secrets used for deployment live in . Any changes to the environment should be made there and then tested and validated by devops folks. diff --git a/docker-compose.yaml b/docker-compose.yaml index 710e5c1..bb585ff 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: container_name: signal_documentation-db restart: always env_file: - - ./.env + - ./.env environment: MYSQL_DATABASE: ${MYSQL_DATABASE} MYSQL_USER: ${MYSQL_USER} @@ -18,11 +18,13 @@ services: ports: - "3306:3306" - webapp: + sdwebapp: + image: ${REGISTRY}signal_documentation-webapp${TAG} build: . env_file: - - ./.env - container_name: signal_documentation-web + - ./.env + container_name: signal_documentation-webapp + restart: on-failure command: sh -c "python3 /usr/src/signal_documentation/src/manage.py migrate --noinput && python3 /usr/src/signal_documentation/src/manage.py collectstatic --noinput && python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/available_geography.json && @@ -44,6 +46,20 @@ services: ports: - "6379:6379" + sdnginx: + image: ${REGISTRY}signal_documentation-nginx${TAG} + build: ./nginx + env_file: + - ./.env + container_name: signal_documentation-nginx + restart: on-failure + volumes: + - ./src/staticfiles:/staticfiles + ports: + - "80:80" + depends_on: + - sdwebapp volumes: mysql: webapp: + static: diff --git a/nginx/default.conf.template b/nginx/default.conf.template index 6d48f14..fef97cc 100644 --- a/nginx/default.conf.template +++ b/nginx/default.conf.template @@ -1,16 +1,17 @@ server { - listen 80; - server_name _; + listen 80; + server_name sdnginx; - location / { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://${APP_HOST}:8000; - } + location /static/ { + autoindex on; + alias /staticfiles/; + } - location /usr/src/signal_documentation { - alias /static/; - } + location / { + proxy_pass http://sdwebapp:8000/; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + } }