From 6f59db9061a82b38d754e715ecfef7dbf78366f1 Mon Sep 17 00:00:00 2001 From: Roman Reznikov Date: Sun, 3 Jul 2022 16:20:45 +0400 Subject: [PATCH] #22: feat: configure MongoDB --- .github/workflows/workflow.yml | 5 ++ README.md | 12 ++++ docker-compose-dev.yml | 39 +++++++++++++ docker-compose.yml | 41 +++++++++++++ requirements/requirements.in | 1 + requirements/requirements.txt | 89 +++++++++++++++++++++++++++++ tests/functional/docker-compose.yml | 21 +++++++ 7 files changed, 208 insertions(+) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index cd3697a..983ad84 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -79,6 +79,11 @@ jobs: export NUGC_REDIS_DEFAULT_CHARSET=utf-8 export NUGC_REDIS_DECODE_RESPONSES=1 export NUGC_REDIS_RETRY_ON_TIMEOUT=1 + export NUGC_MONGODB_USER=test + export NUGC_MONGODB_PASSWORD=yandex + export NUGC_MONGODB_NAME=netflix-ugc + export NUGC_MONGOEXPRESS_LOGIN=admin + export NUGC_MONGOEXPRESS_PASSWORD=pass export NUGC_QUEUE_PROGRESS_NAME=progress-topic export NUGC_QUEUE_PROGRESS_GROUP=progress-group export NUGC_QUEUE_PROGRESS_CONSUMERS=2 diff --git a/README.md b/README.md index d478017..114a852 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,12 @@ NUGC_REDIS_OM_URL=redis://@redis:6379 NUGC_REDIS_DEFAULT_CHARSET=utf-8 NUGC_REDIS_DECODE_RESPONSES=1 NUGC_REDIS_RETRY_ON_TIMEOUT=1 +# MongoDB +NUGC_MONGODB_USER=yandex +NUGC_MONGODB_PASSWORD=netflix +NUGC_MONGODB_NAME=netflix_ugc +NUGC_MONGOEXPRESS_LOGIN=admin +NUGC_MONGOEXPRESS_PASSWORD=pass # Queue NUGC_QUEUE_PROGRESS_NAME=progress-topic NUGC_QUEUE_PROGRESS_GROUP=progress-group @@ -151,6 +157,12 @@ NUGC_REDIS_OM_URL=redis://@redis:6379 NUGC_REDIS_DEFAULT_CHARSET=utf-8 NUGC_REDIS_DECODE_RESPONSES=1 NUGC_REDIS_RETRY_ON_TIMEOUT=1 +# MongoDB +NUGC_MONGODB_USER=yandex +NUGC_MONGODB_PASSWORD=netflix +NUGC_MONGODB_NAME=netflix_ugc +NUGC_MONGOEXPRESS_LOGIN=admin +NUGC_MONGOEXPRESS_PASSWORD=pass # Queue NUGC_QUEUE_PROGRESS_NAME=progress-topic NUGC_QUEUE_PROGRESS_GROUP=progress-group diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 021933e..9022dab 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -27,6 +27,44 @@ services: - "traefik.http.routers.ugc_api.rule=Host(`api-ugc.localhost`,`traefik`)" - "traefik.http.routers.ugc_api.entrypoints=web" + mongodb: + image: mongo:5.0 + restart: unless-stopped + command: mongod --auth + environment: + - MONGO_INITDB_ROOT_USERNAME=${NUGC_MONGODB_USER} + - MONGO_INITDB_ROOT_PASSWORD=${NUGC_MONGODB_PASSWORD} + - MONGO_INITDB_DATABASE=${NUGC_MONGODB_NAME} + - MONGODB_DATA_DIR=/data/db + - MONDODB_LOG_DIR=/dev/null + volumes: + - mongodb_data:/data/db + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongo mongodb:27017/${NUGC_MONGODB_NAME} --quiet || exit 1 + interval: 10s + timeout: 10s + retries: 5 + start_period: 15s + + mongo-express: + image: mongo-express:0.54 + restart: on-failure + ports: + - "8061:8081" + environment: + - ME_CONFIG_MONGODB_SERVER=mongodb + - ME_CONFIG_MONGODB_PORT=27017 + - ME_CONFIG_MONGODB_ENABLE_ADMIN=true + - ME_CONFIG_OPTIONS_EDITORTHEME=default + - ME_CONFIG_MONGODB_AUTH_DATABASE=admin + - ME_CONFIG_MONGODB_ADMINUSERNAME=${NUGC_MONGODB_USER} + - ME_CONFIG_MONGODB_ADMINPASSWORD=${NUGC_MONGODB_PASSWORD} + - ME_CONFIG_BASICAUTH_USERNAME=${NUGC_MONGOEXPRESS_LOGIN} + - ME_CONFIG_BASICAUTH_PASSWORD=${NUGC_MONGOEXPRESS_PASSWORD} + depends_on: + mongodb: + condition: service_healthy + redis: image: redis/redis-stack:7.0.0-RC5 ports: @@ -371,3 +409,4 @@ services: volumes: redis_data: + mongodb_data: diff --git a/docker-compose.yml b/docker-compose.yml index 16d2f97..64eb95c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,8 @@ services: depends_on: redis: condition: service_healthy + mongodb: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.routers.ugc_api.rule=Host(`api-ugc.localhost`,`traefik`)" @@ -39,6 +41,44 @@ services: timeout: 5s retries: 5 + mongodb: + image: mongo:5.0 + restart: unless-stopped + command: mongod --auth + environment: + - MONGO_INITDB_ROOT_USERNAME=${NUGC_MONGODB_USER} + - MONGO_INITDB_ROOT_PASSWORD=${NUGC_MONGODB_PASSWORD} + - MONGO_INITDB_DATABASE=${NUGC_MONGODB_NAME} + - MONGODB_DATA_DIR=/data/db + - MONDODB_LOG_DIR=/dev/null + volumes: + - mongodb_data:/data/db + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongo mongodb:27017/${NUGC_MONGODB_NAME} --quiet || exit 1 + interval: 10s + timeout: 10s + retries: 5 + start_period: 15s + + mongo-express: + image: mongo-express:0.54 + restart: on-failure + ports: + - "8061:8081" + environment: + - ME_CONFIG_MONGODB_SERVER=mongodb + - ME_CONFIG_MONGODB_PORT=27017 + - ME_CONFIG_MONGODB_ENABLE_ADMIN=true + - ME_CONFIG_OPTIONS_EDITORTHEME=default + - ME_CONFIG_MONGODB_AUTH_DATABASE=admin + - ME_CONFIG_MONGODB_ADMINUSERNAME=${NUGC_MONGODB_USER} + - ME_CONFIG_MONGODB_ADMINPASSWORD=${NUGC_MONGODB_PASSWORD} + - ME_CONFIG_BASICAUTH_USERNAME=${NUGC_MONGOEXPRESS_LOGIN} + - ME_CONFIG_BASICAUTH_PASSWORD=${NUGC_MONGOEXPRESS_PASSWORD} + depends_on: + mongodb: + condition: service_healthy + traefik: image: traefik:v2.8 command: @@ -60,3 +100,4 @@ services: volumes: redis_data: + mongodb_data: diff --git a/requirements/requirements.in b/requirements/requirements.in index da9568d..cba1a29 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -22,6 +22,7 @@ aiokafka==0.7.2 aioredis==2.0.1 redis-om==0.0.27 hiredis==2.0.0 +motor==3.0.0 gunicorn==20.1.0 uvloop==0.16.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0d98808..03911df 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -492,6 +492,10 @@ marshmallow-enum==1.5.1 \ --hash=sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58 \ --hash=sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072 # via -r requirements.in +motor==3.0.0 \ + --hash=sha256:3e36d29406c151b61342e6a8fa5e90c00c4723b76e30f11276a4373ea2064b7d \ + --hash=sha256:b076de44970f518177f0eeeda8b183f52eafa557775bfe3294e93bda18867a71 + # via -r requirements.in multidict==6.0.2 \ --hash=sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60 \ --hash=sha256:041b81a5f6b38244b34dc18c7b6aba91f9cdaf854d9a39e5ff0b58e2b5773b9c \ @@ -658,6 +662,91 @@ pylev==1.4.0 \ --hash=sha256:7b2e2aa7b00e05bb3f7650eb506fc89f474f70493271a35c242d9a92188ad3dd \ --hash=sha256:9e77e941042ad3a4cc305dcdf2b2dec1aec2fbe3dd9015d2698ad02b173006d1 # via cleo +pymongo==4.1.1 \ + --hash=sha256:019a4c13ef1d9accd08de70247068671b116a0383adcd684f6365219f29f41cd \ + --hash=sha256:07f50a3b8a3afb086089abcd9ab562fb2a27b63fd7017ca13dfe7b663c8f3762 \ + --hash=sha256:08a619c92769bd7346434dfc331a3aa8dc63bee80ed0be250bb0e878c69a6f3e \ + --hash=sha256:0a3474e6a0df0077a44573727341df6627042df5ca61ea5373c157bb6512ccc7 \ + --hash=sha256:0b8a1c766de29173ddbd316dbd75a97b19a4cf9ac45a39ad4f53426e5df1483b \ + --hash=sha256:0f7e3872fb7b61ec574b7e04302ea03928b670df583f8691cb1df6e54cd42b19 \ + --hash=sha256:17df40753085ccba38a0e150001f757910d66440d9b5deced30ed4cc8b45b6f3 \ + --hash=sha256:298908478d07871dbe17e9ccd37a10a27ad3f37cc1faaf0cc4d205da3c3e8539 \ + --hash=sha256:302ac0f4825501ab0900b8f1a2bb2dc7d28f69c7f15fbc799fb26f9b9ebb1ecb \ + --hash=sha256:303d1b3da2461586379d98b344b529598c8156857285ba5bd156dab1c875d1f6 \ + --hash=sha256:306336dab4537b2343e52ec34017c3051c3aee5a961fff4915ab27f7e6d9b1e9 \ + --hash=sha256:30d35a8855f328a85e5002f0908b24e500efdf8f5f78b73098995ce111baa2a9 \ + --hash=sha256:3139c9ddee379c22a9109a0b3bf4cdb64597db2bbd3909f7a2825b47226977a4 \ + --hash=sha256:32e785c37f6a0e844788c6085ea2c9c0c528348c22cebe91896705a92f2b1b26 \ + --hash=sha256:33a5693e8d1fbb7743b7e867d43c1095652a0c6fedddab6cefe6020bee2ca393 \ + --hash=sha256:35d02603c2318676fca5049cdc722bb2e7a378eaccf139ad767365e0eb3bcdbe \ + --hash=sha256:4516a5ce2beaebddc74d6e304ed520324dda99573c310ef4078284b026f81e93 \ + --hash=sha256:49bb36986f11da2da190a2e777a411c0a28eeb8623850091ea8099b84e3860c7 \ + --hash=sha256:4aa4800530782f7d38aeb169476a5bc692aacc394686f0ca3866e4bb85c9aa3f \ + --hash=sha256:4d1cdece06156542c18b691511a01fe78a694b9fa287ffd8e15680dbf2beeed5 \ + --hash=sha256:4e4d2babb8737d650250d0fa940ffa1b88aa92b8eb399af093734950a1eeca45 \ + --hash=sha256:4fd5c4f25d8d488ee5701c3ec786f52907dca653b47ce8709bcc2bfb0f5506ae \ + --hash=sha256:52c8b7bffd2140818ade2aa28c24cfe47935a7273a3bb976d1d8fb17e716536f \ + --hash=sha256:56b856a459762a3c052987e28ed2bd4b874f0be6671d2cc4f74c4891f47f997a \ + --hash=sha256:571a3e1ef4abeb4ac719ac381f5aada664627b4ee048d9995e93b4bcd0f70601 \ + --hash=sha256:5cae9c935cdc53e4729920543b7d990615a115d85f32144773bc4b2b05144628 \ + --hash=sha256:5d6ef3fa41f3e3be93483a77f81dea8c7ce5ed4411382a31af2b09b9ec5d9585 \ + --hash=sha256:6396f0db060db9d8751167ea08f3a77a41a71cd39236fade4409394e57b377e8 \ + --hash=sha256:69beffb048de19f7c18617b90e38cbddfac20077b1826c27c3fe2e3ef8ac5a43 \ + --hash=sha256:7507439cd799295893b5602f438f8b6a0f483efb00720df1aa33a39102b41bcf \ + --hash=sha256:7aa40509dd9f75c256f0a7533d5e2ccef711dbbf0d91c13ac937d21d76d71656 \ + --hash=sha256:7d69a3d980ecbf7238ab37b9027c87ad3b278bb3742a150fc33b5a8a9d990431 \ + --hash=sha256:7dae2cf84a09329617b08731b95ad1fc98d50a9b40c2007e351438bd119a2f7a \ + --hash=sha256:7f36eacc70849d40ce86c85042ecfcbeab810691b1a3b08062ede32a2d6521ac \ + --hash=sha256:7f55a602d55e8f0feafde533c69dfd29bf0e54645ab0996b605613cda6894a85 \ + --hash=sha256:8357aa727094798f1d831339ecfd8b3e388c01db6015a3cbd51790cb75e39994 \ + --hash=sha256:84dc6bfeaeba98fe93fc837b12f9af4842694cdbde18083f150e80aec3de88f9 \ + --hash=sha256:86b18420f00d5977bda477369ac85e04185ef94046a04ae0d85f5a807d1a8eb4 \ + --hash=sha256:89f32d8450e15b0c11efdc81e2704d68c502c889d48415a50add9fa031144f75 \ + --hash=sha256:8a1de8931cdad8cd12724e12a6167eef8cb478cc3ee5d2c9f4670c934f2975e1 \ + --hash=sha256:8f106468062ac7ff03e3522a66cb7b36c662326d8eb7af1be0f30563740ff002 \ + --hash=sha256:9a4ea87a0401c06b687db29e2ae836b2b58480ab118cb6eea8ac2ef45a4345f8 \ + --hash=sha256:9ee1b019a4640bf39c0705ab65e934cfe6b89f1a8dc26f389fae3d7c62358d6f \ + --hash=sha256:a0d7c6d6fbca62508ea525abd869fca78ecf68cd3bcf6ae67ec478aa37cf39c0 \ + --hash=sha256:a1417cb339a367a5dfd0e50193a1c0e87e31325547a0e7624ee4ff414c0b53b3 \ + --hash=sha256:a35f1937b0560587d478fd2259a6d4f66cf511c9d28e90b52b183745eaa77d95 \ + --hash=sha256:a4a35e83abfdac7095430e1c1476e0871e4b234e936f4a7a7631531b09a4f198 \ + --hash=sha256:a7d1c8830a7bc10420ceb60a256d25ab5b032a6dad12a46af6ab2e470cee9124 \ + --hash=sha256:a938d4d5b530f8ea988afb80817209eabc150c53b8c7af79d40080313a35e470 \ + --hash=sha256:a9a2c377106fe01a57bad0f703653de286d56ee5285ed36c6953535cfa11f928 \ + --hash=sha256:baf7546afd27be4f96f23307d7c295497fb512875167743b14a7457b95761294 \ + --hash=sha256:bb21e2f35d6f09aa4a6df0c716f41e036cfcf05a98323b50294f93085ad775e9 \ + --hash=sha256:bc62ba37bcb42e4146b853940b65a2de31c2962d2b6da9bc3ce28270d13b5c4e \ + --hash=sha256:be3ba736aabf856195199208ed37459408c932940cbccd2dc9f6ff2e800b0261 \ + --hash=sha256:c03eb43d15c8af58159e7561076634d565530aaacaf48cf4e070c3501e88a372 \ + --hash=sha256:c1349331fa743eed4042f9652200e60596f8beb957554acbcbb42aad4272c606 \ + --hash=sha256:c3637cfce519560e2a2579d05eb81e912d109283b8ddc8de46f57ec20d273d92 \ + --hash=sha256:c481cd1af2a77f58f495f7f87c2d715c6f1179d07c1ec927cca1f7977a2d99aa \ + --hash=sha256:c575f9499e5f540e034ff87bef894f031ae613a98b0d1d3afcc1f482527d5f1c \ + --hash=sha256:c604831daf2e7e5979ecd97a90cb8c4a7bae208ff45bc792e32eae09c3281afb \ + --hash=sha256:c759e1e0333664831d8d1d6b26cf59f23f3707758f696c71f506504b33130f81 \ + --hash=sha256:c8a2743dd50629c0222f26c5f55975e45841d985b4b1c7a54b3f03b53de3427d \ + --hash=sha256:cbcac9263f500da94405cc9fc7e7a42a3ba6c2fe88b2cd7039737cba44c66889 \ + --hash=sha256:cce1b7a680653e31ff2b252f19a39f1ded578a35a96c419ddb9632c62d2af7d8 \ + --hash=sha256:cf96799b3e5e2e2f6dbca015f72b28e7ae415ce8147472f89a3704a035d6336d \ + --hash=sha256:d06ed18917dbc7a938c4231cbbec52a7e474be270b2ef9208abb4d5a34f5ceb9 \ + --hash=sha256:d4ba5b4f1a0334dbe673f767f28775744e793fcb9ea57a1d72bc622c9f90e6b4 \ + --hash=sha256:d7b8f25c9b0043cbaf77b8b895814e33e7a3c807a097377c07e1bd49946030d5 \ + --hash=sha256:d86511ef8217822fb8716460aaa1ece31fe9e8a48900e541cb35acb7c35e9e2e \ + --hash=sha256:db8a9cbe965c7343feab2e2bf9a3771f303f8a7ca401dececb6ef28e06b3b18c \ + --hash=sha256:dbe92a8808cefb284e235b8f82933d7d2e24ff929fe5d53f1fd3ca55fced4b58 \ + --hash=sha256:deb83cc9f639045e2febcc8d4306d4b83893af8d895f2ed70aa342a3430b534c \ + --hash=sha256:df9084e06efb3d59608a6a443faa9861828585579f0ae8e95f5a4dab70f1a00f \ + --hash=sha256:dfb89e92746e4a1e0d091cba73d6cc1e16b4094ebdbb14c2e96a80320feb1ad7 \ + --hash=sha256:e13ddfe2ead9540e8773cae098f54c5206d6fcef64846a3e5042db47fc3a41ed \ + --hash=sha256:e4956384340eec7b526149ac126c8aa11d32441cb3ce77a690cb4821d0d0635c \ + --hash=sha256:e6eecd027b6ba5617ea6af3e12e20d578d8f4ad1bf51a9abe69c6fd4835ea532 \ + --hash=sha256:eff9818b7671a55f1ce781398607e0d8c304cd430c0581fbe15b868a7a371c27 \ + --hash=sha256:f0aea377b9dfc166c8fa05bb158c30ee3d53d73f0ed2fc05ba6c638d9563422f \ + --hash=sha256:f1fba193ab2f25849e24caa4570611aa2f80bc1c1ba791851523734b4ed69e43 \ + --hash=sha256:f6db4f00d3baad615e99a865539391243d12b113fb628ebda1d7794ce02d5a10 \ + --hash=sha256:f9405c02af86850e0a8a8ba777b7e7609e0d07bff46adc4f78892cc2d5456018 \ + --hash=sha256:fb4445e3721720c5ca14c0650f35c263b3430e6e16df9d2504618df914b3fb99 + # via motor pyparsing==3.0.9 \ --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc diff --git a/tests/functional/docker-compose.yml b/tests/functional/docker-compose.yml index 575f79f..aaa268d 100644 --- a/tests/functional/docker-compose.yml +++ b/tests/functional/docker-compose.yml @@ -23,11 +23,30 @@ services: depends_on: redis: condition: service_healthy + mongodb: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.routers.ugc_api.rule=Host(`api-ugc.localhost`,`traefik`)" - "traefik.http.routers.ugc_api.entrypoints=ugc" + mongodb: + image: mongo:5.0 + restart: unless-stopped + command: mongod --auth + environment: + - MONGO_INITDB_ROOT_USERNAME=${NUGC_MONGODB_USER} + - MONGO_INITDB_ROOT_PASSWORD=${NUGC_MONGODB_PASSWORD} + - MONGO_INITDB_DATABASE=${NUGC_MONGODB_NAME} + - MONGODB_DATA_DIR=/data/db + - MONDODB_LOG_DIR=/dev/null + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongo mongodb:27017/${NUGC_MONGODB_NAME} --quiet || exit 1 + interval: 10s + timeout: 10s + retries: 5 + start_period: 15s + redis: image: redis/redis-stack-server:7.0.0-RC5 healthcheck: @@ -122,6 +141,8 @@ services: condition: service_started redis: condition: service_started + mongodb: + condition: service_healthy server-auth: condition: service_healthy db-auth: