From 7b93c3dba0b277c712acf393bd225b08b6ee78d0 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 19 Mar 2024 20:41:42 +0400 Subject: [PATCH] chore: Clean up dependencies in setup.py (#4004) chore: setup.py cleanup Signed-off-by: tokoko --- Makefile | 3 +- .../requirements/py3.10-ci-requirements.txt | 45 +++++++++---------- .../requirements/py3.10-requirements.txt | 36 +++++---------- .../requirements/py3.9-ci-requirements.txt | 45 +++++++++---------- .../requirements/py3.9-requirements.txt | 36 +++++---------- setup.py | 16 ++----- 6 files changed, 72 insertions(+), 109 deletions(-) diff --git a/Makefile b/Makefile index 1a1d3b6e92..0eac7e03a2 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,8 @@ build: protos build-java build-docker install-python-ci-dependencies: python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt - COMPILE_GO=true python setup.py develop + pip install --no-deps -e . + python setup.py build_python_protos --inplace lock-python-ci-dependencies: python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index b00b85822c..8f0ef90d77 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -41,8 +41,6 @@ attrs==23.2.0 # bowler # jsonschema # referencing -avro==1.11.3 - # via feast (setup.py) azure-core==1.30.1 # via # azure-identity @@ -63,11 +61,11 @@ black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.60 +boto3==1.34.65 # via # feast (setup.py) # moto -botocore==1.34.60 +botocore==1.34.65 # via # boto3 # moto @@ -126,7 +124,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.4.3 +coverage[toml]==7.4.4 # via pytest-cov cryptography==42.0.5 # via @@ -242,7 +240,7 @@ google-cloud-datastore==2.19.0 # via feast (setup.py) google-cloud-firestore==2.15.0 # via firebase-admin -google-cloud-storage==2.15.0 +google-cloud-storage==2.16.0 # via # feast (setup.py) # firebase-admin @@ -260,7 +258,7 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.10 +great-expectations==0.18.11 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy @@ -333,7 +331,7 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) -importlib-resources==6.1.3 +importlib-resources==6.3.1 # via feast (setup.py) iniconfig==2.0.0 # via pytest @@ -369,7 +367,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.22 +json5==0.9.24 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -401,7 +399,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.9.1 +jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.4 # via jupyterlab @@ -414,7 +412,7 @@ jupyter-server==2.13.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.4 +jupyterlab==4.1.5 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -483,11 +481,11 @@ mypy-extensions==1.0.0 # mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.9.1 +nbclient==0.10.0 # via nbconvert nbconvert==7.16.2 # via jupyter-server -nbformat==5.10.2 +nbformat==5.10.3 # via # great-expectations # jupyter-server @@ -497,7 +495,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.1.1 +notebook==7.1.2 # via great-expectations notebook-shim==0.2.4 # via @@ -583,7 +581,6 @@ prompt-toolkit==3.0.43 # via ipython proto-plus==1.23.0 # via - # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -626,7 +623,7 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==15.0.1 +pyarrow==15.0.2 # via # db-dtypes # feast (setup.py) @@ -750,7 +747,7 @@ pyzmq==25.1.2 # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.33.0 +referencing==0.34.0 # via # jsonschema # jsonschema-specifications @@ -800,7 +797,7 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -s3transfer==0.10.0 +s3transfer==0.10.1 # via boto3 scipy==1.12.0 # via great-expectations @@ -846,7 +843,9 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==1.4.52 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy sqlglot==20.11.0 @@ -855,7 +854,7 @@ stack-data==0.6.3 # via ipython starlette==0.36.3 # via fastapi -substrait==0.14.0 +substrait==0.14.1 # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) @@ -930,7 +929,7 @@ types-pymysql==1.1.0.1 # via feast (setup.py) types-pyopenssl==24.0.0.20240311 # via types-redis -types-python-dateutil==2.8.19.20240311 +types-python-dateutil==2.9.0.20240316 # via # arrow # feast (setup.py) @@ -942,7 +941,7 @@ types-redis==4.6.0.20240311 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.1.0.20240310 +types-setuptools==69.2.0.20240317 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -1021,7 +1020,7 @@ wrapt==1.16.0 # via testcontainers xmltodict==0.13.0 # via moto -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index bbce1ecb7f..e17a588538 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -8,7 +8,6 @@ annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via - # httpx # starlette # watchfiles appdirs==1.4.4 @@ -21,10 +20,7 @@ attrs==23.2.0 bowler==0.9.0 # via feast (setup.py) certifi==2024.2.2 - # via - # httpcore - # httpx - # requests + # via requests charset-normalizer==3.3.2 # via requests click==8.1.7 @@ -48,32 +44,25 @@ fastapi==0.110.0 # via feast (setup.py) fissix==21.11.13 # via bowler -fsspec==2024.2.0 +fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy gunicorn==21.2.0 # via feast (setup.py) h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.4 - # via httpx + # via uvicorn httptools==0.6.1 # via uvicorn -httpx==0.27.0 - # via feast (setup.py) idna==3.6 # via # anyio - # httpx # requests importlib-metadata==6.11.0 # via # dask # feast (setup.py) -importlib-resources==6.1.3 +importlib-resources==6.3.1 # via feast (setup.py) jinja2==3.1.3 # via feast (setup.py) @@ -108,14 +97,11 @@ pandas==2.2.1 # via feast (setup.py) partd==1.4.1 # via dask -proto-plus==1.23.0 - # via feast (setup.py) protobuf==4.25.3 # via # feast (setup.py) # mypy-protobuf - # proto-plus -pyarrow==15.0.1 +pyarrow==15.0.2 # via feast (setup.py) pydantic==2.6.4 # via @@ -136,7 +122,7 @@ pyyaml==6.0.1 # dask # feast (setup.py) # uvicorn -referencing==0.33.0 +referencing==0.34.0 # via # jsonschema # jsonschema-specifications @@ -149,11 +135,11 @@ rpds-py==0.18.0 six==1.16.0 # via python-dateutil sniffio==1.3.1 - # via - # anyio - # httpx + # via anyio sqlalchemy[mypy]==1.4.52 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy starlette==0.36.3 @@ -200,5 +186,5 @@ watchfiles==0.21.0 # via uvicorn websockets==12.0 # via uvicorn -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index de0a507f8c..dc96554431 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -41,8 +41,6 @@ attrs==23.2.0 # bowler # jsonschema # referencing -avro==1.11.3 - # via feast (setup.py) azure-core==1.30.1 # via # azure-identity @@ -63,11 +61,11 @@ black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.60 +boto3==1.34.65 # via # feast (setup.py) # moto -botocore==1.34.60 +botocore==1.34.65 # via # boto3 # moto @@ -126,7 +124,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.4.3 +coverage[toml]==7.4.4 # via pytest-cov cryptography==42.0.5 # via @@ -242,7 +240,7 @@ google-cloud-datastore==2.19.0 # via feast (setup.py) google-cloud-firestore==2.15.0 # via firebase-admin -google-cloud-storage==2.15.0 +google-cloud-storage==2.16.0 # via # feast (setup.py) # firebase-admin @@ -260,7 +258,7 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.10 +great-expectations==0.18.11 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy @@ -341,7 +339,7 @@ importlib-metadata==6.11.0 # nbconvert # sphinx # typeguard -importlib-resources==6.1.3 +importlib-resources==6.3.1 # via feast (setup.py) iniconfig==2.0.0 # via pytest @@ -377,7 +375,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.22 +json5==0.9.24 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -409,7 +407,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.9.1 +jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.4 # via jupyterlab @@ -422,7 +420,7 @@ jupyter-server==2.13.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.4 +jupyterlab==4.1.5 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -491,11 +489,11 @@ mypy-extensions==1.0.0 # mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.9.1 +nbclient==0.10.0 # via nbconvert nbconvert==7.16.2 # via jupyter-server -nbformat==5.10.2 +nbformat==5.10.3 # via # great-expectations # jupyter-server @@ -505,7 +503,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.1.1 +notebook==7.1.2 # via great-expectations notebook-shim==0.2.4 # via @@ -591,7 +589,6 @@ prompt-toolkit==3.0.43 # via ipython proto-plus==1.23.0 # via - # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -634,7 +631,7 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==15.0.1 +pyarrow==15.0.2 # via # db-dtypes # feast (setup.py) @@ -758,7 +755,7 @@ pyzmq==25.1.2 # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.33.0 +referencing==0.34.0 # via # jsonschema # jsonschema-specifications @@ -810,7 +807,7 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -s3transfer==0.10.0 +s3transfer==0.10.1 # via boto3 scipy==1.12.0 # via great-expectations @@ -856,7 +853,9 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==1.4.52 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy sqlglot==20.11.0 @@ -865,7 +864,7 @@ stack-data==0.6.3 # via ipython starlette==0.36.3 # via fastapi -substrait==0.14.0 +substrait==0.14.1 # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) @@ -940,7 +939,7 @@ types-pymysql==1.1.0.1 # via feast (setup.py) types-pyopenssl==24.0.0.20240311 # via types-redis -types-python-dateutil==2.8.19.20240311 +types-python-dateutil==2.9.0.20240316 # via # arrow # feast (setup.py) @@ -952,7 +951,7 @@ types-redis==4.6.0.20240311 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.1.0.20240310 +types-setuptools==69.2.0.20240317 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -1035,7 +1034,7 @@ wrapt==1.16.0 # via testcontainers xmltodict==0.13.0 # via moto -zipp==3.17.0 +zipp==3.18.1 # via # importlib-metadata # importlib-resources diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index ac0954c167..f2228ade02 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -8,7 +8,6 @@ annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via - # httpx # starlette # watchfiles appdirs==1.4.4 @@ -21,10 +20,7 @@ attrs==23.2.0 bowler==0.9.0 # via feast (setup.py) certifi==2024.2.2 - # via - # httpcore - # httpx - # requests + # via requests charset-normalizer==3.3.2 # via requests click==8.1.7 @@ -48,33 +44,26 @@ fastapi==0.110.0 # via feast (setup.py) fissix==21.11.13 # via bowler -fsspec==2024.2.0 +fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy gunicorn==21.2.0 # via feast (setup.py) h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.4 - # via httpx + # via uvicorn httptools==0.6.1 # via uvicorn -httpx==0.27.0 - # via feast (setup.py) idna==3.6 # via # anyio - # httpx # requests importlib-metadata==6.11.0 # via # dask # feast (setup.py) # typeguard -importlib-resources==6.1.3 +importlib-resources==6.3.1 # via feast (setup.py) jinja2==3.1.3 # via feast (setup.py) @@ -109,14 +98,11 @@ pandas==2.2.1 # via feast (setup.py) partd==1.4.1 # via dask -proto-plus==1.23.0 - # via feast (setup.py) protobuf==4.25.3 # via # feast (setup.py) # mypy-protobuf - # proto-plus -pyarrow==15.0.1 +pyarrow==15.0.2 # via feast (setup.py) pydantic==2.6.4 # via @@ -137,7 +123,7 @@ pyyaml==6.0.1 # dask # feast (setup.py) # uvicorn -referencing==0.33.0 +referencing==0.34.0 # via # jsonschema # jsonschema-specifications @@ -150,11 +136,11 @@ rpds-py==0.18.0 six==1.16.0 # via python-dateutil sniffio==1.3.1 - # via - # anyio - # httpx + # via anyio sqlalchemy[mypy]==1.4.52 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy starlette==0.36.3 @@ -202,7 +188,7 @@ watchfiles==0.21.0 # via uvicorn websockets==12.0 # via uvicorn -zipp==3.17.0 +zipp==3.18.1 # via # importlib-metadata # importlib-resources diff --git a/setup.py b/setup.py index 92d7356411..b32d03ed77 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,6 @@ "pandas>=1.4.3,<3", # Higher than 4.23.4 seems to cause a seg fault "protobuf>=4.24.0,<5.0.0", - "proto-plus>=1.20.0,<2", "pyarrow>=4", "pydantic>=2.0.0", "pygments>=2.12.0,<3", @@ -70,8 +69,6 @@ # https://github.com/dask/dask/issues/10996 "dask>=2021.1.0,<2024.3.0", "bowler", # Needed for automatic repo upgrades - # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). - "httpx>=0.23.3", "importlib-resources>=6.0.0,<7", "importlib_metadata>=6.8.0,<7", ] @@ -163,11 +160,12 @@ "black>=22.6.0,<23", "isort>=5,<6", "grpcio-testing>=1.56.2,<2", + # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). + "httpx>=0.23.3", "minio==7.1.0", "mock==2.0.0", "moto<5", "mypy>=1.4.1", - "avro==1.10.0", "urllib3>=1.25.4,<3", "psutil==5.9.0", "py>=1.11.0", # https://github.com/pytest-dev/pytest/issues/10420 @@ -215,14 +213,8 @@ + GRPCIO_REQUIRED ) - -# rtd builds fail because of mysql not being installed in their environment. -# We can add mysql there, but it's not strictly needed. This will be faster for builds. -DOCS_REQUIRED = CI_REQUIRED.copy() -for _r in MYSQL_REQUIRED: - DOCS_REQUIRED.remove(_r) - -DEV_REQUIRED = ["grpcio-testing~=1.0"] + CI_REQUIRED +DOCS_REQUIRED = CI_REQUIRED +DEV_REQUIRED = CI_REQUIRED # Get git repo root directory repo_root = str(pathlib.Path(__file__).resolve().parent)