diff --git a/.dockerignore b/.dockerignore index 5f760dd00..ad81ec054 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,7 +9,7 @@ **/.webassets-cache **/__pycache__ .webpack_cache -.cache +.eslintcache # MacOS file **/.DS_Store @@ -74,6 +74,9 @@ env/ # Versioning files .git +# github workflow files +.github + # Local DBs dump.rdb test.db diff --git a/.github/workflows/docker-ci-tests.yml b/.github/workflows/docker-ci-tests.yml deleted file mode 100644 index bf055aeeb..000000000 --- a/.github/workflows/docker-ci-tests.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: 'Pytest on docker' - -on: - push: - branches: ['main'] - pull_request: - branches: ['main'] - paths: - - '**.py' - - '**.js' - - '**.scss' - - '**.jinja2' - - 'requirements/base.txt' - - 'requirements/test.txt' - - '.github/workflows/docker-ci-tests.yml' - - 'Dockerfile' - - 'pyproject.toml' - - '.eslintrc.js' - - 'docker-compose.yml' - - 'docker/compose/services.yml' - - 'docker/entrypoints/ci-test.sh' - - 'docker/initdb/test.sh' - - 'package.json' -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Cache npm - uses: actions/cache@v3 - with: - path: .cache/.npm - key: docker-npm - - name: Cache node_modules - uses: actions/cache@v3 - with: - path: node_modules - key: docker-node_modules-${{ hashFiles('package-lock.json') }} - - name: Cache pip - uses: actions/cache@v3 - with: - path: .cache/pip - key: docker-pip - - name: Cache .local - uses: actions/cache@v3 - with: - path: .cache/.local - key: docker-user-local - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build funnel-test image - id: build-funnel-test - uses: docker/build-push-action@v4 - with: - context: . - file: ci.Dockerfile - tags: funnel-test:latest - load: true - push: false - - name: Run Tests - run: make docker-ci-test - - name: Upload coverage report to Coveralls - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.github_token }} - path-to-lcov: coverage/funnel.lcov - flag-name: docker-3.11 diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml new file mode 100644 index 000000000..07752e4d6 --- /dev/null +++ b/.github/workflows/pytest-docker.yml @@ -0,0 +1,92 @@ +name: 'Pytest Docker' + +on: + push: + paths: + - '**.py' + - '**.js' + - '**.scss' + - '**.jinja2' + - '.flaskenv' + - '.testenv' + - 'requirements/base.txt' + - 'requirements/test.txt' + - '.github/workflows/pytest-docker.yml' + - 'Dockerfile' + - 'Makefile' + - 'pyproject.toml' + - 'docker-compose.ci.yml' + - 'docker/entrypoints/ci.sh' + - 'docker/initdb.sh' + - 'package.json' + - 'package-lock.json' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build funnel-ci image + id: build-funnel-ci + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + target: ci + tags: funnel-ci + load: true + push: false + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Set permissions for coverage + run: docker compose -f docker-compose.ci.yml run --rm -u root --entrypoint "" --no-deps --quiet-pull main chown -R 1000:1000 coverage + - name: Test with pytest + run: docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --no-log-prefix --exit-code-from main + - name: Upload coverage report to Coveralls + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.github_token }} + path-to-lcov: coverage/funnel.lcov + flag-name: docker-3.11 + - name: Set short git commit SHA + if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} + id: sha + run: | + short_sha=$(git rev-parse --short ${{ github.sha }}) + echo "short=$short_sha" >> "$GITHUB_OUTPUT" + - name: Login to Docker Hub + if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GitHub Container Registry + if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GHCR_WRITE_TOKEN }} + - name: Build & push funnel image + if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} + id: build-funnel + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + target: app + tags: | + hasgeek/funnel:${{ github.ref_name }} + hasgeek/funnel:sha-${{ steps.sha.outputs.short }} + ghcr.io/hasgeek/funnel:${{ github.ref_name }} + ghcr.io/hasgeek/funnel:sha-${{ steps.sha.outputs.short }} + push: true + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index ca291c340..f17f71085 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,6 @@ __pycache__ .nox monkeytype.sqlite3 -.ci-cache -.cache # MacOS file .DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d0ab4a4c..74cf86a81 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -204,10 +204,10 @@ repos: # hooks: # - id: dockerfile_lint # files: .*Dockerfile.* - # - repo: https://github.com/hadolint/hadolint - # rev: v2.12.1-beta - # hooks: - # - id: hadolint-docker + - repo: https://github.com/hadolint/hadolint + rev: v2.12.1-beta + hooks: + - id: hadolint-docker - repo: https://github.com/IamTheFij/docker-pre-commit rev: v3.0.1 hooks: diff --git a/.testenv b/.testenv index f60567fe7..9a9ba3b9f 100644 --- a/.testenv +++ b/.testenv @@ -46,9 +46,7 @@ FLASK_IMAGE_URL_DOMAINS='["images.example.com"]' FLASK_IMAGE_URL_SCHEMES='["https"]' FLASK_SES_NOTIFICATION_TOPICS=null # Per app config -APP_FUNNEL_SITE_ID=hasgeek-test APP_FUNNEL_SERVER_NAME=funnel.test:3002 APP_FUNNEL_SHORTLINK_DOMAIN=f.test:3002 APP_FUNNEL_DEFAULT_DOMAIN=funnel.test APP_FUNNEL_UNSUBSCRIBE_DOMAIN=bye.test -APP_SHORTLINK_SITE_ID=shortlink-test diff --git a/Dockerfile b/Dockerfile index 9693027e2..53ce203b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,112 +1,70 @@ # syntax=docker/dockerfile:1.4 -FROM nikolaik/python-nodejs:python3.11-nodejs20-bullseye as base +# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md -# https://github.com/zalando/postgres-operator/blob/master/docker/logical-backup/Dockerfile -# https://stackoverflow.com/questions/68465355/what-is-the-meaning-of-set-o-pipefail-in-bash-script -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +FROM node:lts-alpine as assets +USER node +WORKDIR /home/node/app +RUN mkdir -pv /home/node/app/funnel/static/build /home/node/app/funnel/static/build_cache +COPY --chown=node:node package.json package-lock.json ./ +RUN --mount=type=cache,target=/home/node/.npm/,uid=1000,gid=1000 npm ci +COPY --chown=node:node ./funnel/assets/ ./funnel/assets/ +COPY --chown=node:node webpack.config.js .eslintrc.js ./ +RUN --mount=type=cache,target=/home/node/app/.webpack_cache/,uid=1000,gid=1000 \ + --mount=type=cache,target=/home/node/app/funnel/static/build_cache/,uid=1000,gid=1000 \ + cp -R funnel/static/build_cache funnel/static/build \ + && npm run build \ + && cp -R funnel/static/build funnel/static/build_cache \ + && cp -R funnel/static/build funnel/static/built -LABEL Name=Funnel -LABEL Version=0.1 +FROM python:3.11-bullseye as app +LABEL maintainer="Hasgeek" +RUN chsh -s /usr/sbin/nologin root +# hadolint ignore=DL3008 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + apt-get update -yqq \ + && apt-get install -yqq --no-install-recommends supervisor curl \ + && apt-get autoclean -yqq \ + && apt-get autoremove -yqq \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -pv /var/log/supervisor +RUN addgroup --gid 1000 funnel && adduser --uid 1000 --gid 1000 funnel +ENV PATH "$PATH:/home/funnel/.local/bin" +USER funnel +# hadolint ignore=DL4006 +RUN curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | /bin/bash -s -- -y +ENV PATH "/home/funnel/.cargo/bin:$PATH" +WORKDIR /home/funnel/app -USER pn -RUN \ - mkdir -pv /home/pn/.cache/pip /home/pn/.npm /home/pn/tmp /home/pn/app /home/pn/app/coverage && \ - chown -R pn:pn /home/pn/.cache /home/pn/.npm /home/pn/tmp /home/pn/app /home/pn/app/coverage -EXPOSE 3000 -WORKDIR /home/pn/app +COPY --chown=funnel:funnel Makefile Makefile +COPY --chown=funnel:funnel requirements/base.txt requirements/base.txt +RUN mkdir -pv /home/funnel/.cache/pip +# hadolint ignore=DL3013,DL3042 +RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python && pip install --upgrade uwsgi -ENV PATH "$PATH:/home/pn/.local/bin" +COPY --chown=funnel:funnel . . +COPY --from=assets --chown=funnel:funnel /home/node/app/funnel/static/built/ funnel/static/build +RUN mkdir -pv /home/funnel/app/logs +ENTRYPOINT [ "uwsgi", "--ini" ] -FROM base as devtest_base +FROM app as ci USER root -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -yqq && \ - apt-get install -yqq --no-install-recommends lsb-release && \ - sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \ - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ - apt-get update -yqq && apt-get upgrade -yqq && \ - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends firefox-esr postgresql-client-15 && \ - cd /tmp/ && \ - curl -fsSL $(curl -fsSL https://api.github.com/repos/mozilla/geckodriver/releases/latest | grep browser_download_url | grep 'linux64.tar.gz\"'| grep -o 'http.*\.gz') > gecko.tar.gz && \ - tar -xvzf gecko.tar.gz && \ - rm gecko.tar.gz && \ - chmod +x geckodriver && \ - mv geckodriver /usr/local/bin && \ - apt-get autoclean -yqq && \ - apt-get autoremove -yqq && \ - cd /home/pn/app -USER pn - -FROM base as assets -COPY --chown=pn:pn package.json package.json -COPY --chown=pn:pn package-lock.json package-lock.json -RUN --mount=type=cache,target=/root/.npm \ - --mount=type=cache,target=/home/pn/.npm,uid=1000,gid=1000 npm ci -COPY --chown=pn:pn ./funnel/assets ./funnel/assets -COPY --chown=pn:pn .eslintrc.js .eslintrc.js -COPY --chown=pn:pn webpack.config.js webpack.config.js -RUN --mount=type=cache,target=/root/.npm \ - --mount=type=cache,target=/home/pn/.npm,uid=1000,gid=1000 npm run build - -FROM base as dev_assets -COPY --chown=pn:pn package.json package.json -COPY --chown=pn:pn package-lock.json package-lock.json -RUN --mount=type=cache,target=/root/.npm \ - --mount=type=cache,target=/home/pn/.npm,uid=1000,gid=1000 npm install -COPY --chown=pn:pn ./funnel/assets ./funnel/assets -COPY --chown=pn:pn .eslintrc.js .eslintrc.js -COPY --chown=pn:pn webpack.config.js webpack.config.js -RUN --mount=type=cache,target=/root/.npm \ - --mount=type=cache,target=/home/pn/.npm,uid=1000,gid=1000 npx webpack --mode development --progress - -FROM base as deps -COPY --chown=pn:pn Makefile Makefile -RUN make deps-editable -COPY --chown=pn:pn requirements/base.txt requirements/base.txt -RUN --mount=type=cache,target=/home/pn/.cache/pip,uid=1000,gid=1000 \ - pip install --upgrade pip && \ - pip install --use-pep517 -r requirements/base.txt - -FROM devtest_base as test_deps -COPY --chown=pn:pn Makefile Makefile -RUN make deps-editable -COPY --chown=pn:pn requirements/base.txt requirements/base.txt -COPY --chown=pn:pn requirements/test.txt requirements/test.txt -RUN --mount=type=cache,target=/home/pn/.cache/pip,uid=1000,gid=1000 pip install --use-pep517 -r requirements/test.txt - -FROM devtest_base as dev_deps -COPY --chown=pn:pn Makefile Makefile -RUN make deps-editable -COPY --chown=pn:pn requirements requirements -RUN --mount=type=cache,target=/home/pn/.cache/pip,uid=1000,gid=1000 pip install --use-pep517 -r requirements/dev.txt -COPY --from=dev_assets --chown=pn:pn /home/pn/app/node_modules /home/pn/app/node_modules - -FROM deps as production -COPY --chown=pn:pn . . -COPY --chown=pn:pn --from=assets /home/pn/app/funnel/static /home/pn/app/funnel/static -ENTRYPOINT ["uwsgi", "--ini"] - -FROM production as supervisor -USER root -RUN \ - apt-get update -yqq && \ - apt-get install -yqq --no-install-recommends supervisor && \ - apt-get autoclean -yqq && \ - apt-get autoremove -yqq && \ - mkdir -pv /var/log/supervisor -COPY ./docker/supervisord/supervisord.conf /etc/supervisor/supervisord.conf -# COPY ./docker/uwsgi/emperor.ini /etc/uwsgi/emperor.ini -ENTRYPOINT ["/usr/bin/supervisord"] - -FROM test_deps as test -ENV PWD=/home/pn/app -COPY --chown=pn:pn . . - -COPY --chown=pn:pn --from=assets /home/pn/app/funnel/static /home/pn/app/funnel/static -ENTRYPOINT ["/home/pn/app/docker/entrypoints/ci-test.sh"] -FROM dev_deps as dev -RUN --mount=type=cache,target=/home/pn/.cache/pip,uid=1000,gid=1000 cp -R /home/pn/.cache/pip /home/pn/tmp/.cache_pip -RUN mv /home/pn/tmp/.cache_pip /home/pn/.cache/pip -COPY --chown=pn:pn --from=dev_assets /home/pn/app/funnel/static /home/pn/app/funnel/static +RUN mkdir -pv /home/funnel/app/coverage && chown -R 1000:1000 /home/funnel/.cache /home/funnel/app/coverage +# hadolint ignore=DL3008,DL4006,SC2046 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + apt-get update -yqq \ + && apt-get install -yqq --no-install-recommends xvfb firefox-esr \ + && apt-get autoclean -yqq \ + && apt-get autoremove -yqq \ + && rm -rf /var/lib/apt/lists/* \ + && curl -fsSL $(curl -fsSL https://api.github.com/repos/mozilla/geckodriver/releases/latest \ + | grep browser_download_url \ + | grep 'linux64.tar.gz\"' \ + | grep -o 'http.*\.gz') \ + | tar -xvz -C /usr/local/bin +USER funnel +ENV PYTHONUNBUFFERED=1 +ENV GITHUB_ACTIONS=true +COPY --chown=funnel:funnel requirements/base.txt requirements/test.txt ./requirements/ +RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python-test +ENTRYPOINT [ "/home/funnel/app/docker/entrypoints/ci.sh" ] diff --git a/Makefile b/Makefile index 2d35005d0..2a8047683 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,8 @@ docker-base-devtest: docker buildx build -f docker/images/bases.Dockerfile --target base-devtest --tag hasgeek/funnel-base-devtest . docker-ci-test: - COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 BUILDKIT_PROGRESS=plain \ - docker compose --profile test up --quiet-pull --no-attach db-test --no-attach redis-test --no-log-prefix + docker compose -f docker-compose.ci.yml run --rm -u root --entrypoint "" --no-deps --quiet-pull main chown -R 1000:1000 coverage && \ + docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --no-log-prefix --exit-code-from main docker-dev: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 \ diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 000000000..7feb45466 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,66 @@ +name: funnel-ci +services: + main: + image: funnel-ci + build: + context: . + target: ci + links: + - db + - redis + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + restart: no + volumes: + - ./coverage:/home/funnel/app/coverage + environment: + - REDIS_HOST=redis + - FLASK_RUN_HOST=host.docker.internal + - DB_HOST=host.docker.internal + - DB_FUNNEL=funnel + - FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg://funnel@db/funnel + - FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg://funnel@db/geoname + extra_hosts: + - funnel.test:127.0.0.1 + - f.test:127.0.0.1 + - bye.test:127.0.0.1 + redis: + image: redis:latest + expose: + - 6379 + restart: always + healthcheck: + test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping'] + volumes: + - redis:/data + db: + image: postgres:latest + restart: always + user: postgres + environment: + - POSTGRES_HOST_AUTH_METHOD=trust + - POSTGRES_USER=postgres + healthcheck: + interval: 5s + timeout: 5s + retries: 5 + test: ['CMD-SHELL', 'psql funnel'] + volumes: + - db:/var/lib/postgresql/data + - ./docker/initdb.sh:/docker-entrypoint-initdb.d/initdb.sh:ro +volumes: + redis: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + o: 'uid=999,gid=999' # uid:gid is 999:999 for redis + db: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + o: 'uid=999,gid=999' # uid:gid is 999:999 for redis diff --git a/docker-compose.postfix.yml b/docker-compose.postfix.yml new file mode 100644 index 000000000..4edc53398 --- /dev/null +++ b/docker-compose.postfix.yml @@ -0,0 +1,20 @@ +services: + main: + environment: + - FLASK_MAIL_SERVER=postfix + - FLASK_MAIL_PORT=25 + shortlink: + environment: + - FLASK_MAIL_SERVER=postfix + - FLASK_MAIL_PORT=25 + postfix: + image: wildwildangel/postfix-relay:latest + env_file: + # https://github.com/sjinks/docker-alpine-postfix-relay + - .env.postfix + environment: + - SERVER_HOSTNAME=postfix + volumes: + - postfix:/var/spool/postfix +volumes: + postfix: diff --git a/docker-compose.yml b/docker-compose.yml index 3c9b0d1d9..ff6d27589 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,210 +1,55 @@ name: funnel -x-postgres: &postgres - image: postgres:latest - restart: always - user: postgres - environment: - - POSTGRES_HOST_AUTH_METHOD=trust - - POSTGRES_USER=postgres - expose: - - 5432 - healthcheck: - interval: 5s - timeout: 5s - retries: 5 -x-redis: &redis - image: redis:latest - expose: - - 6379 - restart: always - healthcheck: - test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping'] -x-app: &app - extends: - file: docker/compose/services.yml - service: funnel-prod - build: - context: . - target: production - image: funnel - profiles: - - production - depends_on: - - redis - environment: - - REDIS_HOST=redis -x-test: &test-app - extends: - file: docker/compose/services.yml - service: funnel - image: funnel-test - profiles: - - test - build: - context: . - dockerfile: ci.Dockerfile - working_dir: /home/pn/app - user: pn - volumes: - - ./.cache/.npm:/home/pn/.npm - - ./.cache/node_modules:/home/pn/app/node_modules - - ./.cache/pip:/home/pn/.cache/pip - - ./.cache/.local:/home/pn/.local - - ./coverage:/home/pn/app/coverage - restart: 'no' services: - app: - <<: *app - volumes: - - ./instance/settings.py:/home/pn/app/instance/settings.py - - ./docker/uwsgi/funnel.ini:/home/pn/funnel.ini:ro - command: ../funnel.ini + main: &main-app + image: funnel + build: + context: . + target: app + logging: + driver: json-file + options: + max-size: 200k + max-file: 10 + links: + - redis + depends_on: + - redis + env_file: + - .env.production + environment: + - REDIS_HOST=redis + - FLASK_RUN_HOST=host.docker.internal + command: ['docker/uwsgi/funnel.ini'] expose: - 6400 ports: - 6400:6400 - pre-test: - <<: *test-app - user: root - entrypoint: ['/home/pn/app/docker/entrypoints/ci-pre-test.sh'] - test: - <<: *test-app - depends_on: - pre-test: - condition: service_completed_successfully - redis-test: - condition: service_healthy - db-test: - condition: service_healthy - links: - - db-test - - redis-test - environment: - - REDIS_HOST=redis-test - - DB_HOST=db-test - - FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg://funnel@db-test/funnel_testing - - FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg://funnel@db-test/geoname_testing - db-test: - <<: *postgres - profiles: - - test - volumes: - - postgres_test:/var/lib/postgresql/data - - ./docker/initdb/test.sh:/docker-entrypoint-initdb.d/test.sh:ro - healthcheck: - test: ['CMD-SHELL', 'psql funnel_testing'] - redis-test: - <<: *redis - profiles: - - test - volumes: - - redis_test:/data - dev: - extends: - file: docker/compose/services.yml - service: funnel - image: funnel-dev - container_name: funnel-dev - profiles: - - dev - - dev-no-watch - build: - context: . - target: dev - depends_on: - redis-dev: - condition: service_healthy - db-dev: - condition: service_healthy - working_dir: /home/pn/app - entrypoint: /home/pn/dev-entrypoint.sh + shortlink: + <<: *main-app + command: ['docker/uwsgi/shortlink.ini'] + expose: + - 6410 ports: - - 3000:3000 - links: - - db-dev - - redis-dev - volumes: - # https://stackoverflow.com/questions/43844639/how-do-i-add-cached-or-delegated-into-a-docker-compose-yml-volumes-list - # https://forums.docker.com/t/what-happened-to-delegated-cached-ro-and-other-flags/105097/2 - - pip_cache:/home/pn/.cache/pip:delegated - - .:/home/pn/app - - node_modules:/home/pn/app/node_modules - - ./docker/entrypoints/dev.sh:/home/pn/dev-entrypoint.sh:ro - - ./instance/settings.py:/home/pn/app/instance/settings.py - environment: - - DB_HOST=db-dev - - POSTGRES_USER_HOST=funnel@db-dev - - REDIS_HOST=redis-dev - healthcheck: - test: bash -c '[[ "$$(curl -o /dev/null -s -w "%{http_code}\n" http://funnel.test:3000)" == "200" ]]' - interval: 30s - timeout: 1m - retries: 10 - start_period: 30s - asset-watcher: - extends: - file: docker/compose/services.yml - service: funnel - image: funnel-dev-asset-watcher - container_name: funnel-dev-asset-watcher - profiles: - - dev - build: - context: . - target: dev-assets - working_dir: /home/pn/app - entrypoint: npx webpack --mode development --watch - volumes: - - .:/home/pn/app - - node_modules:/home/pn/app/node_modules - environment: - - NODE_ENV=development - depends_on: - dev: - condition: service_healthy - healthcheck: - test: bash -c "[[ -f /home/pn/app/funnel/static/build/manifest.json ]]" - interval: 10s - timeout: 30s - retries: 60 - start_period: 1m - db-dev: - <<: *postgres - profiles: - - dev - - dev-no-watch - volumes: - - postgres_dev:/var/lib/postgresql/data - - ./docker/initdb/dev.sh:/docker-entrypoint-initdb.d/dev.sh:ro - healthcheck: - test: ['CMD-SHELL', 'psql funnel'] - redis-dev: - <<: *redis + - 6410:6410 + rq: + <<: *main-app + entrypoint: ['supervisord'] + command: ['-c', 'docker/supervisor/rq.conf'] + expose: [] + ports: [] + bash: + <<: *main-app + entrypoint: [''] profiles: - - dev - - dev-no-watch - volumes: - - redis_dev:/data + - maintenance redis: - <<: *redis - profiles: - - production + image: redis:latest + expose: + - 6379 + restart: always + healthcheck: + test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping'] volumes: - redis:/data -x-tmpfs: &tmpfs - driver: local - driver_opts: - type: tmpfs - device: tmpfs - o: 'uid=999,gid=999' # uid:gid is 999:999 for both postgres and redis - volumes: - node_modules: - pip_cache: - postgres_dev: - redis_dev: redis: - postgres_test: - <<: *tmpfs - redis_test: - <<: *tmpfs diff --git a/docker/.npmrc b/docker/.npmrc deleted file mode 100644 index 7a650526c..000000000 --- a/docker/.npmrc +++ /dev/null @@ -1,4 +0,0 @@ -audit = false -fund = false -loglevel = warn -update-notifier = false diff --git a/docker/compose/services.yml b/docker/compose/services.yml deleted file mode 100644 index 4bc90056e..000000000 --- a/docker/compose/services.yml +++ /dev/null @@ -1,20 +0,0 @@ -services: - funnel: - logging: - driver: json-file - options: - max-size: 200k - max-file: 10 - extra_hosts: - - 'funnel.test:127.0.0.1' - - 'f.test:127.0.0.1' - environment: - - FLASK_RUN_HOST=0.0.0.0 - funnel-prod: - extends: - file: services.yml - service: funnel - build: - target: production - links: - - redis diff --git a/docker/entrypoints/ci-pre-test.sh b/docker/entrypoints/ci-pre-test.sh deleted file mode 100755 index 29a558f0a..000000000 --- a/docker/entrypoints/ci-pre-test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# https://github.com/docker/for-mac/issues/5480 - -chown -R pn:pn /home/pn/.npm /home/pn/.cache /home/pn/.cache/pip /home/pn/app \ - /home/pn/app/coverage /home/pn/.local diff --git a/docker/entrypoints/ci-test.sh b/docker/entrypoints/ci-test.sh deleted file mode 100755 index 4da374e54..000000000 --- a/docker/entrypoints/ci-test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -make install-test -pytest --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts db-test | awk '/STREAM/ { print $1}'),$(getent ahosts redis-test | awk '/STREAM/ { print $1}') --gherkin-terminal-reporter -vv --showlocals --cov=funnel -coverage lcov -o coverage/funnel.lcov diff --git a/docker/entrypoints/ci.sh b/docker/entrypoints/ci.sh new file mode 100755 index 000000000..23386f5eb --- /dev/null +++ b/docker/entrypoints/ci.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +pytest \ + --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') \ + --gherkin-terminal-reporter -vv --showlocals --cov=funnel $@ +coverage lcov -o coverage/funnel.lcov diff --git a/docker/entrypoints/dev.sh b/docker/entrypoints/dev.sh deleted file mode 100755 index 9135c8ff1..000000000 --- a/docker/entrypoints/dev.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -if [ "$(psql -XtA -U postgres -h $DB_HOST funnel -c "select count(*) from information_schema.tables where table_schema = 'public';")" = "0" ]; then - flask dbcreate - flask db stamp -fi - -./devserver.py diff --git a/docker/images/bases.Dockerfile b/docker/images/bases.Dockerfile deleted file mode 100644 index 9e27b0cf7..000000000 --- a/docker/images/bases.Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# syntax=docker/dockerfile:1.4 - -# Dockerfile syntax & features documentation: -# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md - -FROM nikolaik/python-nodejs:python3.11-nodejs20-bullseye as base - -# https://github.com/zalando/postgres-operator/blob/master/docker/logical-backup/Dockerfile -# https://stackoverflow.com/questions/68465355/what-is-the-meaning-of-set-o-pipefail-in-bash-script -SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] - -STOPSIGNAL SIGINT -ENV PATH "$PATH:/home/pn/.local/bin" - -# Install postgresql-client-15 -USER root:root -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -y \ - && apt-get install -y --no-install-recommends lsb-release \ - && echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ - && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ - && apt-get update -y && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends postgresql-client-15 \ - && apt-get purge -y lsb-release -RUN mkdir -pv /var/cache/funnel && chown -R pn:pn /var/cache/funnel -USER pn:pn - -FROM base as base-devtest -# Install firefox & geckodriver -USER root:root -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -y \ - && apt-get upgrade -y \ - && echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends firefox-esr \ - && cd /tmp/ \ - && curl -fsSL $(curl -fsSL https://api.github.com/repos/mozilla/geckodriver/releases/latest | grep browser_download_url | grep 'linux64.tar.gz\"'| grep -o 'http.*\.gz') > gecko.tar.gz \ - && tar -xvzf gecko.tar.gz \ - && rm gecko.tar.gz \ - && chmod +x geckodriver \ - && mv geckodriver /usr/local/bin -USER pn:pn diff --git a/docker/initdb.sh b/docker/initdb.sh new file mode 100755 index 000000000..cb87cbb0b --- /dev/null +++ b/docker/initdb.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +psql -c "create user funnel;" +psql -c "create database funnel;" +psql -c "create database geoname;" +psql funnel << $$ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE EXTENSION IF NOT EXISTS unaccent; +CREATE EXTENSION IF NOT EXISTS pgcrypto; +$$ +psql geoname << $$ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE EXTENSION IF NOT EXISTS unaccent; +CREATE EXTENSION IF NOT EXISTS pgcrypto; +$$ +psql -c "grant all privileges on database funnel to funnel;" +psql -c "grant all privileges on database geoname to funnel;" +psql funnel -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" +psql geoname -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" diff --git a/docker/initdb/dev.sh b/docker/initdb/dev.sh deleted file mode 100755 index 4cc3bf36d..000000000 --- a/docker/initdb/dev.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -e - -psql -c "create user funnel;" -psql -c "create database funnel;" -psql -c "create database geoname;" -psql -c "create database funnel_testing;" -psql -c "create database geoname_testing;" -psql funnel << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ -psql geoname << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ - -psql funnel_testing << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ -psql geoname_testing << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ - -psql -c "grant all privileges on database funnel to funnel;" -psql -c "grant all privileges on database geoname to funnel;" -psql funnel -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" -psql geoname -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" - -psql -c "grant all privileges on database funnel_testing to funnel;" -psql -c "grant all privileges on database geoname_testing to funnel;" -psql funnel_testing -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" -psql geoname_testing -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" diff --git a/docker/initdb/test.sh b/docker/initdb/test.sh deleted file mode 100755 index 2d7e0962f..000000000 --- a/docker/initdb/test.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -e - -psql -c "create user funnel;" -psql -c "create database funnel_testing;" -psql -c "create database geoname_testing;" -psql funnel_testing << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ -psql geoname_testing << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ -psql -c "grant all privileges on database funnel_testing to funnel;" -psql -c "grant all privileges on database geoname_testing to funnel;" -psql funnel_testing -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" -psql geoname_testing -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" diff --git a/docker/supervisor/rq.conf b/docker/supervisor/rq.conf new file mode 100644 index 000000000..f9245dbcd --- /dev/null +++ b/docker/supervisor/rq.conf @@ -0,0 +1,29 @@ +[supervisord] +nodaemon=true +logfile=/dev/null +logfile_maxbytes=0 + +[program:rq-funnel] +; Point the command to the specific rqworker command you want to run. +; If you use virtualenv, be sure to point it to +; /path/to/virtualenv/bin/rqworker +; Also, you probably want to include a settings module to configure this +; worker. For more info on that, see http://python-rq.org/docs/workers/ +command=flask rq worker funnel +process_name=%(program_name)s-%(process_num)s +user=funnel + +; If you want to run more than one worker instance, increase this +numprocs=2 + +; This is the directory from which RQ is ran. Be sure to point this to the +; directory where your source code is importable from +directory=/home/funnel/app + +; RQ requires the TERM signal to perform a warm shutdown. If RQ does not die +; within 10 seconds, supervisor will forcefully kill it +stopsignal=TERM + +; These are up to you +autostart=true +autorestart=true diff --git a/docker/supervisord/supervisord.conf b/docker/supervisord/supervisord.conf deleted file mode 100644 index afead1552..000000000 --- a/docker/supervisord/supervisord.conf +++ /dev/null @@ -1,28 +0,0 @@ -; supervisor config file - -[unix_http_server] -file=/var/run/supervisor.sock ; (the path to the socket file) -chmod=0700 ; sockef file mode (default 0700) - -[supervisord] -logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) -pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) -childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) - -; the below section must remain in the config file for RPC -; (supervisorctl/web interface) to work, additional interfaces may be -; added by defining them in separate rpcinterface: sections -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL for a unix socket - -; The [include] section can just contain the "files" setting. This -; setting can list multiple files (separated by whitespace or -; newlines). It can also contain wildcards. The filenames are -; interpreted as relative to this file. Included files *cannot* -; include files themselves. - -[include] -files = /etc/supervisor/conf.d/*.conf diff --git a/docker/uwsgi/funnel.ini b/docker/uwsgi/funnel.ini index 857590750..218eaa486 100644 --- a/docker/uwsgi/funnel.ini +++ b/docker/uwsgi/funnel.ini @@ -1,12 +1,12 @@ [uwsgi] -socket = 0.0.0.0:6400 +http = 0.0.0.0:6400 processes = 6 threads = 2 master = true uid = funnel gid = funnel -chdir = /home/pn/app +chdir = /home/funnel/app wsgi-file = wsgi.py callable = application buffer-size = 24000 -pidfile = /home/pn/%n.pid +pidfile = /home/funnel/%n.pid diff --git a/docker/uwsgi/shortlink.ini b/docker/uwsgi/shortlink.ini new file mode 100644 index 000000000..75ba8c985 --- /dev/null +++ b/docker/uwsgi/shortlink.ini @@ -0,0 +1,12 @@ +[uwsgi] +http = 0.0.0.0:6410 +processes = 2 +threads = 2 +master = true +uid = funnel +gid = funnel +chdir = /home/funnel/app +wsgi-file = wsgi.py +callable = shortlinkapp +buffer-size = 24000 +pidfile = /home/funnel/%n.pid diff --git a/requirements/test.in b/requirements/test.in index 5b43a9994..d6037d9fd 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -12,12 +12,13 @@ pytest-bdd pytest-cov pytest-dotenv pytest-env +pytest-github-actions-annotate-failures pytest-rerunfailures pytest-selenium>=4.0.1 pytest-socket requests-mock respx -selenium<4.10 # Selenium 4.10.0 breaks pytest-selenium 4.0.1 +selenium>=4.14 sttable tomlkit typeguard diff --git a/requirements/test.txt b/requirements/test.txt index c9de37b30..df30efef9 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,4 @@ -# SHA1:371c24d3103594f1897af063fbc0ccf09017932c +# SHA1:590ef9cef00d0620cbea0915e666e73e585a58bc # # This file is autogenerated by pip-compile-multi # To update, run: @@ -48,6 +48,7 @@ pytest==7.4.2 # pytest-cov # pytest-dotenv # pytest-env + # pytest-github-actions-annotate-failures # pytest-html # pytest-metadata # pytest-rerunfailures @@ -66,6 +67,8 @@ pytest-dotenv==0.5.2 # via -r requirements/test.in pytest-env==1.0.1 # via -r requirements/test.in +pytest-github-actions-annotate-failures==0.2.0 + # via -r requirements/test.in pytest-html==4.0.2 # via pytest-selenium pytest-metadata==3.0.0 @@ -82,7 +85,7 @@ requests-mock==1.11.0 # via -r requirements/test.in respx==0.20.2 # via -r requirements/test.in -selenium==4.9.1 +selenium==4.14.0 # via # -r requirements/test.in # pytest-selenium diff --git a/sample.env b/sample.env index 159905cbc..d6e43c0a9 100644 --- a/sample.env +++ b/sample.env @@ -24,13 +24,16 @@ FLASK_ENV=development FLASK_DEBUG=1 # Flask-DebugToolbar (optional) is useful for dev, but MUST NOT BE enabled in production FLASK_DEBUG_TB_ENABLED=true +FLASK_DEBUG_TB_INTERCEPT_REDIRECTS=false # --- Domain configuration (these must point to 127.0.0.1 in /etc/hosts in dev and test) # Funnel app's server name (Hasgeek uses 'hasgeek.com' in production) +# Use funnel.test:6400 for docker APP_FUNNEL_SERVER_NAME=funnel.test:3000 # Funnel app's default domain when running without a HTTP context APP_FUNNEL_DEFAULT_DOMAIN=funnel.test # Shortlink domain (Hasgeek uses 'has.gy' in production) +# Use f.test:6410 for docker FLASK_SHORTLINK_DOMAIN=f.test:3000 # Optional unsubscribe URL domain (Hasgeek uses 'bye.li' in production) # https://bye.li/* redirects to https://hasgeek.com/account/notifications/bye/* @@ -96,7 +99,7 @@ FLASK_STATSD_TAGS=, # FLASK_STATSD_FORM_LOG=true # --- Redis Queue and Redis cache (use separate dbs to isolate) -# Redis server host +# Redis server host. Use redis for docker. REDIS_HOST=localhost # RQ and cache FLASK_RQ_REDIS_URL=redis://${REDIS_HOST}:6379/1 @@ -105,14 +108,18 @@ FLASK_CACHE_TYPE=flask_caching.backends.RedisCache FLASK_CACHE_REDIS_URL=redis://${REDIS_HOST}:6379/0 # --- Database configuration +# Use host.docker.internal to connect to host system on docker +# Use postgres for test and dev setup DB_HOST=localhost +DB_FUNNEL=funnel # Main app database -FLASK_SQLALCHEMY_DATABASE_URI='postgresql+psycopg:///funnel' +FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg:///${DB_FUNNEL} # Geoname database (the use of `__` creates a dict and sets a key in the dict) -FLASK_SQLALCHEMY_BINDS__geoname='postgresql+psycopg:///geoname' +FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg:///geoname # --- Email configuration # SMTP mail server ('localhost' if Postfix is configured as a relay email server) +# Use postfix for docker FLASK_MAIL_SERVER=localhost # If not using localhost, SMTP will need authentication # Port number (25 is default, but 587 is more likely for non-localhost) @@ -175,6 +182,12 @@ FLASK_LOG_TELEGRAM_APIKEY=null # Optional settings: # FLASK_LOG_TELEGRAM_THREADID (if the chat has topic threads, use a specific thread) # FLASK_LOG_TELEGRAM_LEVEL=NOTSET, DEBUG, INFO, WARNING (default), ERROR, CRITICAL +# Overrides for metrics being sent to telegraf +# Use host.docker.internal if running statsd on host machine on docker +# STATSD_HOST=127.0.0.1 +# STATSD_PORT=8125 +# STATSD_IPV6=False +# STATSD_TAGS=False # --- Hasgeek app integrations # Imgee image server