From f8af1246b95b8f483e9d70703bc7a9a2b134a354 Mon Sep 17 00:00:00 2001 From: gazwald Date: Wed, 24 Aug 2022 08:13:19 +1000 Subject: [PATCH 1/3] Update requirements --- deployment/requirements.txt | 10 +-- docs/requirements.txt | 16 ++-- master/requirements.txt | 26 +++--- requirements.test.txt | 168 ++++++++++++++++++------------------ 4 files changed, 111 insertions(+), 109 deletions(-) diff --git a/deployment/requirements.txt b/deployment/requirements.txt index 40644f2f..73857862 100644 --- a/deployment/requirements.txt +++ b/deployment/requirements.txt @@ -4,26 +4,26 @@ # # pip-compile ./deployment/requirements.in # -attrs==21.4.0 +attrs==22.1.0 # via # cattrs # jsii -aws-cdk-aws-lambda-python-alpha==2.35.0a0 +aws-cdk-aws-lambda-python-alpha==2.38.1a0 # via -r ./deployment/requirements.in -aws-cdk-lib==2.35.0 +aws-cdk-lib==2.38.1 # via # -r ./deployment/requirements.in # aws-cdk-aws-lambda-python-alpha cattrs==22.1.0 # via jsii -constructs==10.1.70 +constructs==10.1.84 # via # -r ./deployment/requirements.in # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib exceptiongroup==1.0.0rc8 # via cattrs -jsii==1.63.2 +jsii==1.65.0 # via # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib diff --git a/docs/requirements.txt b/docs/requirements.txt index 849e1ef4..77f14389 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,11 +2,11 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile requirements.in +# pip-compile ./docs/requirements.in # certifi==2022.6.15 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via json-schema-for-humans @@ -18,13 +18,13 @@ idna==3.3 # via requests jinja2==3.1.2 # via json-schema-for-humans -json-schema-for-humans==0.41.6 - # via -r requirements.in +json-schema-for-humans==0.41.8 + # via -r ./docs/requirements.in markdown2==2.4.3 # via json-schema-for-humans markupsafe==2.1.1 # via jinja2 -marshmallow==3.17.0 +marshmallow==3.17.1 # via # dataclasses-json # marshmallow-enum @@ -34,7 +34,7 @@ mypy-extensions==0.4.3 # via typing-inspect packaging==21.3 # via marshmallow -pygments==2.12.0 +pygments==2.13.0 # via json-schema-for-humans pyparsing==3.0.9 # via packaging @@ -46,7 +46,7 @@ requests==2.28.1 # via json-schema-for-humans typing-extensions==4.3.0 # via typing-inspect -typing-inspect==0.7.1 +typing-inspect==0.8.0 # via dataclasses-json -urllib3==1.26.11 +urllib3==1.26.12 # via requests diff --git a/master/requirements.txt b/master/requirements.txt index 8b3213c8..cebac801 100644 --- a/master/requirements.txt +++ b/master/requirements.txt @@ -2,35 +2,35 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile requirements.in +# pip-compile ./master/requirements.in # -boto3[crt]==1.24.47 - # via -r requirements.in -boto3-stubs[dynamodb]==1.24.47 - # via -r requirements.in -botocore==1.27.47 +boto3[crt]==1.24.58 + # via -r ./master/requirements.in +boto3-stubs[dynamodb]==1.24.58 + # via -r ./master/requirements.in +botocore==1.27.58 # via # boto3 # s3transfer -botocore-stubs==1.27.47 +botocore-stubs==1.27.58 # via boto3-stubs click==8.1.3 - # via -r requirements.in + # via -r ./master/requirements.in jmespath==1.0.1 # via # boto3 # botocore -mypy-boto3-dynamodb==1.24.36.post1 +mypy-boto3-dynamodb==1.24.55.post1 # via boto3-stubs -pydantic==1.9.1 - # via -r requirements.in +pydantic==1.9.2 + # via -r ./master/requirements.in python-dateutil==2.8.2 # via botocore s3transfer==0.6.0 # via boto3 six==1.16.0 # via python-dateutil -types-awscrt==0.13.14 +types-awscrt==0.14.0 # via botocore-stubs types-s3transfer==0.6.0.post4 # via boto3-stubs @@ -39,5 +39,5 @@ typing-extensions==4.3.0 # boto3-stubs # mypy-boto3-dynamodb # pydantic -urllib3==1.26.11 +urllib3==1.26.12 # via botocore diff --git a/requirements.test.txt b/requirements.test.txt index 6051b129..b1a0607f 100644 --- a/requirements.test.txt +++ b/requirements.test.txt @@ -2,156 +2,157 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile requirements.test.in +# pip-compile ./requirements.test.in # astroid==2.11.7 # via pylint -attrs==21.4.0 +attrs==22.1.0 # via - # -r ./deployment/requirements.txt + # -r ././deployment/requirements.txt # cattrs # jsii # pytest -aws-cdk-aws-lambda-python-alpha==2.35.0a0 - # via -r ./deployment/requirements.txt -aws-cdk-lib==2.35.0 +aws-cdk-aws-lambda-python-alpha==2.38.1a0 + # via -r ././deployment/requirements.txt +aws-cdk-lib==2.38.1 # via - # -r ./deployment/requirements.txt + # -r ././deployment/requirements.txt # aws-cdk-aws-lambda-python-alpha black==22.6.0 - # via -r requirements.test.in -boto3[crt]==1.24.46 + # via -r ./requirements.test.in +boto3[crt]==1.24.58 # via - # -r ./master/requirements.txt + # -r ././master/requirements.txt # moto -boto3-stubs[dynamodb]==1.24.46 - # via -r ./master/requirements.txt -botocore==1.27.46 +boto3-stubs[dynamodb]==1.24.58 + # via -r ././master/requirements.txt +botocore==1.27.58 # via - # -r ./master/requirements.txt + # -r ././master/requirements.txt # boto3 # moto # s3transfer -botocore-stubs==1.27.46 +botocore-stubs==1.27.58 # via - # -r ./master/requirements.txt + # -r ././master/requirements.txt # boto3-stubs build==0.8.0 # via pip-tools cattrs==22.1.0 # via - # -r ./deployment/requirements.txt + # -r ././deployment/requirements.txt # jsii certifi==2022.6.15 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # requests cffi==1.15.1 # via cryptography -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # requests click==8.1.3 # via - # -r ./docs/requirements.txt - # -r ./master/requirements.txt + # -r ././docs/requirements.txt + # -r ././master/requirements.txt # black # json-schema-for-humans # pip-tools -constructs==10.1.70 +constructs==10.1.84 # via - # -r ./deployment/requirements.txt + # -r ././deployment/requirements.txt # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib -coverage[toml]==6.4.3 +coverage[toml]==6.4.4 # via pytest-cov cryptography==37.0.4 # via moto dataclasses-json==0.5.7 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # json-schema-for-humans dill==0.3.5.1 # via pylint -docker==5.0.3 +docker==6.0.0 # via moto exceptiongroup==1.0.0rc8 # via - # -r ./deployment/requirements.txt + # -r ././deployment/requirements.txt # cattrs htmlmin==0.1.12 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # json-schema-for-humans idna==3.3 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # requests iniconfig==1.1.1 # via pytest isort==5.10.1 # via - # -r requirements.test.in + # -r ./requirements.test.in # pylint jinja2==3.1.2 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # json-schema-for-humans # moto jmespath==1.0.1 # via - # -r ./master/requirements.txt + # -r ././master/requirements.txt # boto3 # botocore -jsii==1.63.2 +jsii==1.65.0 # via - # -r ./deployment/requirements.txt + # -r ././deployment/requirements.txt # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib # constructs -json-schema-for-humans==0.41.6 - # via -r ./docs/requirements.txt +json-schema-for-humans==0.41.8 + # via -r ././docs/requirements.txt lazy-object-proxy==1.7.1 # via astroid markdown2==2.4.3 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # json-schema-for-humans markupsafe==2.1.1 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # jinja2 # moto -marshmallow==3.17.0 +marshmallow==3.17.1 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # dataclasses-json # marshmallow-enum marshmallow-enum==1.5.1 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # dataclasses-json mccabe==0.7.0 # via pylint -moto[dynamodb]==3.1.17 - # via -r requirements.test.in +moto[dynamodb]==4.0.0 + # via -r ./requirements.test.in mypy==0.971 - # via -r requirements.test.in -mypy-boto3-dynamodb==1.24.36.post1 + # via -r ./requirements.test.in +mypy-boto3-dynamodb==1.24.55.post1 # via - # -r ./master/requirements.txt + # -r ././master/requirements.txt # boto3-stubs mypy-extensions==0.4.3 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # black # mypy # typing-inspect packaging==21.3 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # build + # docker # marshmallow # pytest pathspec==0.9.0 @@ -161,7 +162,7 @@ pathspec==0.9.0 pep517==0.13.0 # via build pip-tools==6.8.0 - # via -r requirements.test.in + # via -r ./requirements.test.in platformdirs==2.5.2 # via # black @@ -170,7 +171,7 @@ pluggy==1.0.0 # via pytest publication==0.0.3 # via - # -r ./deployment/requirements.txt + # -r ././deployment/requirements.txt # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib # constructs @@ -179,48 +180,48 @@ py==1.11.0 # via pytest pycparser==2.21 # via cffi -pydantic==1.9.1 - # via -r ./master/requirements.txt -pygments==2.12.0 +pydantic==1.9.2 + # via -r ././master/requirements.txt +pygments==2.13.0 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # json-schema-for-humans pylint==2.14.5 - # via -r requirements.test.in + # via -r ./requirements.test.in pyparsing==3.0.9 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # packaging pytest==7.1.2 # via - # -r requirements.test.in + # -r ./requirements.test.in # pytest-cov # pytest-mock pytest-cov==3.0.0 - # via -r requirements.test.in + # via -r ./requirements.test.in pytest-mock==3.8.2 - # via -r requirements.test.in + # via -r ./requirements.test.in python-dateutil==2.8.2 # via - # -r ./deployment/requirements.txt - # -r ./master/requirements.txt + # -r ././deployment/requirements.txt + # -r ././master/requirements.txt # botocore # jsii # moto pytz==2021.3 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # json-schema-for-humans # moto pyyaml==6.0 # via - # -r ./docs/requirements.txt - # -r requirements.test.in + # -r ././docs/requirements.txt + # -r ./requirements.test.in # json-schema-for-humans # yamllint requests==2.28.1 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # docker # json-schema-for-humans # moto @@ -229,12 +230,12 @@ responses==0.21.0 # via moto s3transfer==0.6.0 # via - # -r ./master/requirements.txt + # -r ././master/requirements.txt # boto3 six==1.16.0 # via - # -r ./deployment/requirements.txt - # -r ./master/requirements.txt + # -r ././deployment/requirements.txt + # -r ././master/requirements.txt # python-dateutil tomli==2.0.1 # via @@ -245,43 +246,44 @@ tomli==2.0.1 # pep517 # pylint # pytest -tomlkit==0.11.2 +tomlkit==0.11.4 # via pylint typeguard==2.13.3 # via - # -r ./deployment/requirements.txt + # -r ././deployment/requirements.txt # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib # constructs # jsii -types-awscrt==0.13.14 +types-awscrt==0.14.0 # via - # -r ./master/requirements.txt + # -r ././master/requirements.txt # botocore-stubs types-s3transfer==0.6.0.post4 # via - # -r ./master/requirements.txt + # -r ././master/requirements.txt # boto3-stubs typing-extensions==4.3.0 # via - # -r ./deployment/requirements.txt - # -r ./docs/requirements.txt - # -r ./master/requirements.txt + # -r ././deployment/requirements.txt + # -r ././docs/requirements.txt + # -r ././master/requirements.txt # boto3-stubs # jsii # mypy # mypy-boto3-dynamodb # pydantic # typing-inspect -typing-inspect==0.7.1 +typing-inspect==0.8.0 # via - # -r ./docs/requirements.txt + # -r ././docs/requirements.txt # dataclasses-json -urllib3==1.26.11 +urllib3==1.26.12 # via - # -r ./docs/requirements.txt - # -r ./master/requirements.txt + # -r ././docs/requirements.txt + # -r ././master/requirements.txt # botocore + # docker # requests # responses websocket-client==1.3.3 @@ -295,7 +297,7 @@ wrapt==1.14.1 xmltodict==0.13.0 # via moto yamllint==1.27.1 - # via -r requirements.test.in + # via -r ./requirements.test.in # The following packages are considered to be unsafe in a requirements file: # pip From a41bf95630aa827851a4f1fcc427e38aa6088bab Mon Sep 17 00:00:00 2001 From: gazwald Date: Wed, 24 Aug 2022 08:14:57 +1000 Subject: [PATCH 2/3] Adding cachetools dependency --- master/requirements.in | 2 ++ master/requirements.txt | 4 ++++ requirements.test.txt | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/master/requirements.in b/master/requirements.in index 04675351..6e698ea9 100644 --- a/master/requirements.in +++ b/master/requirements.in @@ -2,3 +2,5 @@ pydantic boto3[crt] boto3-stubs[dynamodb] click +cachetools +types-cachetools diff --git a/master/requirements.txt b/master/requirements.txt index cebac801..5c780a8e 100644 --- a/master/requirements.txt +++ b/master/requirements.txt @@ -14,6 +14,8 @@ botocore==1.27.58 # s3transfer botocore-stubs==1.27.58 # via boto3-stubs +cachetools==5.2.0 + # via -r ./master/requirements.in click==8.1.3 # via -r ./master/requirements.in jmespath==1.0.1 @@ -32,6 +34,8 @@ six==1.16.0 # via python-dateutil types-awscrt==0.14.0 # via botocore-stubs +types-cachetools==5.2.1 + # via -r ./master/requirements.in types-s3transfer==0.6.0.post4 # via boto3-stubs typing-extensions==4.3.0 diff --git a/requirements.test.txt b/requirements.test.txt index b1a0607f..6b618391 100644 --- a/requirements.test.txt +++ b/requirements.test.txt @@ -38,6 +38,8 @@ botocore-stubs==1.27.58 # boto3-stubs build==0.8.0 # via pip-tools +cachetools==5.2.0 + # via -r ././master/requirements.txt cattrs==22.1.0 # via # -r ././deployment/requirements.txt @@ -259,6 +261,8 @@ types-awscrt==0.14.0 # via # -r ././master/requirements.txt # botocore-stubs +types-cachetools==5.2.1 + # via -r ././master/requirements.txt types-s3transfer==0.6.0.post4 # via # -r ././master/requirements.txt From b9db0cb814b2d59de829a176c1882bf71122d121 Mon Sep 17 00:00:00 2001 From: gazwald Date: Wed, 24 Aug 2022 08:40:15 +1000 Subject: [PATCH 3/3] Add caching to Dynamodb scan command --- master/storage/backends/dynamodb.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/master/storage/backends/dynamodb.py b/master/storage/backends/dynamodb.py index 8b890da2..600e8350 100644 --- a/master/storage/backends/dynamodb.py +++ b/master/storage/backends/dynamodb.py @@ -1,14 +1,14 @@ import logging -import os from decimal import Decimal +from threading import Lock from typing import Any, Mapping, Optional, Sequence, Union +import cachetools from boto3.dynamodb.conditions import Attr from boto3.session import Session from mypy_boto3_dynamodb.service_resource import DynamoDBServiceResource, Table from mypy_boto3_dynamodb.type_defs import ( GetItemOutputTableTypeDef, - PutItemOutputTableTypeDef, ScanOutputTableTypeDef, UpdateItemOutputTableTypeDef, ) @@ -37,6 +37,8 @@ ], ] +# TODO: maxsize=number of active protocols +CACHE = cachetools.TTLCache(maxsize=3, ttl=60) DEFAULT_SESSION = Session() DEFAULT_TABLE_NAME = f"{APP_NAME}-{DEPLOYMENT_ENVIRONMENT}" @@ -120,6 +122,11 @@ def get_servers(self, game: Optional[str] = None) -> list[Server]: return servers + @cachetools.cached( + cache=CACHE, + key=cachetools.keys.methodkey, + lock=Lock(), + ) def _scan(self, game: Optional[str] = None) -> list[Item]: response: ScanOutputTableTypeDef if game: