From 6adaca658b801bd2c27599c83fc5408b6631f5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Fu=CC=88rst?= Date: Wed, 9 Jun 2021 23:36:44 +0200 Subject: [PATCH 1/6] chore(git): use sophicistaced ignore list Used a virtual environment and thought it would be useful to employ a more sophisticated and automatically generated list of files to ignore. --- .gitignore | 334 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 323 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 31214358..ace0893e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,3 @@ -.idea/ -.vscode/ -__pycache__/ - -*.DS_Store - # log log/ @@ -11,11 +5,329 @@ log/ kontaktdaten.json zeitspanne.json -#building: -installer/ +# Created by https://www.toptal.com/developers/gitignore/api/linux,macos,windows,jetbrains+all,visualstudiocode,python +# Edit at https://www.toptal.com/developers/gitignore?templates=linux,macos,windows,jetbrains+all,visualstudiocode,python + +### JetBrains+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class -# pyinstaller -# temporary building files: -build/ +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ dist/ +downloads/ +eggs/ +.eggs/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +# .env +.env/ +.venv/ +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# operating system-related files +# file properties cache/storage on macOS +*.DS_Store +# thumbnail cache on Windows +Thumbs.db + +# profiling data +.prof + + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +### Windows ### +# Windows thumbnail cache files +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/linux,macos,windows,jetbrains+all,visualstudiocode,python + From e48e9ef2152253bdf9deff06a2b5fc7e63514f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Fu=CC=88rst?= Date: Wed, 9 Jun 2021 23:38:04 +0200 Subject: [PATCH 2/6] feat(docker): support use of remote Chrome instance Allowing to specify a remote instance of Google Chrome allows to run a headless instance of Google Chrome in another container through `selenium/standalone-chrome`. --- tools/its.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/its.py b/tools/its.py index d36db2b5..4775b868 100644 --- a/tools/its.py +++ b/tools/its.py @@ -13,7 +13,7 @@ from selenium.webdriver import ActionChains -from selenium.webdriver import Chrome +from selenium.webdriver import Chrome, Remote from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC @@ -182,8 +182,6 @@ def get_chromedriver_path(self): def get_chromedriver(self, headless): chrome_options = Options() - - # deaktiviere Selenium Logging chrome_options.add_argument('disable-infobars') chrome_options.add_experimental_option('useAutomationExtension', False) @@ -204,7 +202,11 @@ def get_chromedriver(self, headless): chrome_options.headless = headless - return Chrome(self.get_chromedriver_path(), options=chrome_options) + chrome_remote_address = os.getenv("VACCIPY_CHROME_REMOTE_ADDRESS") + if chrome_remote_address: + return Remote(chrome_remote_address, options=chrome_options) + else: + return Chrome(self.get_chromedriver_path(), options=chrome_options) def driver_enter_code(self, driver, plz_impfzentrum): """ From addfe1f08a190fb416be91015ca53ca12713a6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Fu=CC=88rst?= Date: Thu, 10 Jun 2021 01:02:20 +0200 Subject: [PATCH 3/6] refactor(chrome-driver): remove deprecated experimental option In `selenium/standalone-chrome:91`, Google Chrome reports the option `useAutomationExtension` to be deprecated. --- tools/its.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/its.py b/tools/its.py index 4775b868..68371bf2 100644 --- a/tools/its.py +++ b/tools/its.py @@ -184,7 +184,6 @@ def get_chromedriver(self, headless): # deaktiviere Selenium Logging chrome_options.add_argument('disable-infobars') - chrome_options.add_experimental_option('useAutomationExtension', False) chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"]) chrome_options.add_experimental_option('excludeSwitches', ['enable-logging']) From 4d2d94b95614f45f53ee49d99cd4df5e2f8b984f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Fu=CC=88rst?= Date: Thu, 10 Jun 2021 01:03:54 +0200 Subject: [PATCH 4/6] ci(docker): add configuration for Docker Compose Allows to run vaccipy through Docker with an instance of Google Chrome as provided by `selenium/standalone-chrome:91`. --- .dockerignore | 16 ++++++++++++++++ Dockerfile | 9 +++++++++ docker-compose.yaml | 21 +++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..447b939f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.github +.idea +docs +images +specs +venv +.dockerignore +.gitignore +Dockerfile +docker-compose.yaml +FAQ.md +gui.py +impfzentren_gruppiert.md +LICENSE +README.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2ef1ff49 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.9.5-buster + +WORKDIR vaccipy + +ADD . . + +RUN python -m pip install -r requirements.txt + +CMD python3 main.py search -f kontaktdaten.json -r diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 00000000..e1c7fefe --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,21 @@ +version: "3.9" +services: + google-chrome: + environment: + # Necessary to allow connections other than local ones. + - JAVA_OPTS=-Dwebdriver.chrome.whitelistedIps= + image: selenium/standalone-chrome:91.0 + ports: + - "4444:4444" + vaccipy: + build: . + depends_on: + - google-chrome + environment: + - VACCIPY_CHROME_REMOTE_ADDRESS=http://google-chrome:4444/wd/hub + image: vaccipy + volumes: + - type: bind + read_only: true + source: ~/kontaktdaten.json + target: /vaccipy/kontaktdaten.json From f937fee22eed9f4999cc0981e64273cf95f0fa1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Fu=CC=88rst?= Date: Thu, 10 Jun 2021 13:05:40 +0200 Subject: [PATCH 5/6] ci(docker): allow to customize arguments The Docker deployment now allows to customize the arguments passed to vaccipy. Also, the path to a pre-defined `kontaktdaten.json` can now be customized through an environment variable. --- .env | 2 ++ Dockerfile | 2 +- docker-compose.yaml | 6 ++++-- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 00000000..6d556158 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +VACCIPY_ARGUMENTS= +VACCIPY_CONTACT_ADDRESS_PATH=~/kontaktdaten.json \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2ef1ff49..68db45af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,4 +6,4 @@ ADD . . RUN python -m pip install -r requirements.txt -CMD python3 main.py search -f kontaktdaten.json -r +ENTRYPOINT python3 main.py search -f kontaktdaten.json -r diff --git a/docker-compose.yaml b/docker-compose.yaml index e1c7fefe..fddbc46f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,13 +2,15 @@ version: "3.9" services: google-chrome: environment: - # Necessary to allow connections other than local ones. + # Necessary to allow connections other than from the local address. - JAVA_OPTS=-Dwebdriver.chrome.whitelistedIps= image: selenium/standalone-chrome:91.0 ports: - "4444:4444" vaccipy: build: . + # Arguments passed to vaccipy. + command: ${VACCIPY_ARGUMENTS} depends_on: - google-chrome environment: @@ -17,5 +19,5 @@ services: volumes: - type: bind read_only: true - source: ~/kontaktdaten.json + source: ${VACCIPY_CONTACT_ADDRESS_PATH} target: /vaccipy/kontaktdaten.json From b663372f07f5fd673170f2d3722e102512023a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Fu=CC=88rst?= Date: Thu, 10 Jun 2021 13:12:24 +0200 Subject: [PATCH 6/6] ci(docker): wait for Google Chrome to start --- Dockerfile | 6 +++++- docker-compose.yaml | 2 -- wait-for-google-chrome.sh | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 wait-for-google-chrome.sh diff --git a/Dockerfile b/Dockerfile index 68db45af..e3713d48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,12 @@ FROM python:3.9.5-buster WORKDIR vaccipy +RUN apt-get update && apt-get install -y jq + ADD . . RUN python -m pip install -r requirements.txt -ENTRYPOINT python3 main.py search -f kontaktdaten.json -r +RUN chmod +x ./wait-for-google-chrome.sh + +ENTRYPOINT ./wait-for-google-chrome.sh python3 main.py search -f kontaktdaten.json -r diff --git a/docker-compose.yaml b/docker-compose.yaml index fddbc46f..1aafb5e3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,8 +11,6 @@ services: build: . # Arguments passed to vaccipy. command: ${VACCIPY_ARGUMENTS} - depends_on: - - google-chrome environment: - VACCIPY_CHROME_REMOTE_ADDRESS=http://google-chrome:4444/wd/hub image: vaccipy diff --git a/wait-for-google-chrome.sh b/wait-for-google-chrome.sh new file mode 100644 index 00000000..d16ac8ff --- /dev/null +++ b/wait-for-google-chrome.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Taken from https://github.com/SeleniumHQ/docker-selenium#using-a-bash-script-to-wait-for-the-grid. + +set -e + +cmd="$@" + +while ! curl -sSL "http://google-chrome:4444/wd/hub/status" 2>&1 \ + | jq -r '.value.ready' 2>&1 | grep "true" >/dev/null; do + echo 'Waiting for Google Chrome to start...' + sleep 1 +done + +exec $cmd