diff --git a/docs/macros.py b/docs/macros.py index df9971b1a..3c0f63739 100644 --- a/docs/macros.py +++ b/docs/macros.py @@ -11,14 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os from github import Github def define_env(env): @env.macro def github_releases(owner, repo): - github_client = Github() + # due to rate limits, only get this data on release + release_env = os.environ.get("GITHUB_JOB", "dev") + if release_env != "job_deploy_mkdocs_github_pages": + return "----\r\n" + + github_client = Github(retry=0, timeout=5) repo = github_client.get_repo("{}/{}".format(owner, repo)) output = "----\r\n" for release in repo.get_releases(): diff --git a/environment.yml b/environment.yml index ec71b703d..f32cd41de 100644 --- a/environment.yml +++ b/environment.yml @@ -19,32 +19,31 @@ channels: - defaults dependencies: - python>=3.9,<3.12 - - jinja2==3.1.3 + - jinja2>=3.1.3 - pytest==7.4.0 - pytest-mock==3.11.1 - pytest-cov==4.1.0 - pylint==2.17.4 - - pip==23.1.2 + - pip>=23.1.2 - turbodbc==4.11.0 - numpy>=1.23.4 - - pandas>=1.5.2,<3.0.0 - oauthlib>=3.2.2 - cryptography>=38.0.3 - - azure-identity==1.12.0 - - azure-storage-file-datalake==12.12.0 - - azure-keyvault-secrets==4.7.0 - - azure-mgmt-storage==21.0.0 - - boto3==1.28.2 - - pyodbc==4.0.39 - - fastapi==0.110.0 - - httpx==0.24.1 + - azure-identity>=1.12.0 + - azure-storage-file-datalake>=12.12.0 + - azure-keyvault-secrets>=4.7.0 + - azure-mgmt-storage>=21.0.0 + - boto3>=1.28.2 + - pyodbc>=4.0.39 + - fastapi>=0.110.0 + - httpx>=0.24.1 - pyspark>=3.3.0,<3.6.0 - delta-spark>=2.2.0,<3.3.0 - grpcio>=1.48.1 - grpcio-status>=1.48.1 - googleapis-common-protos>=1.56.4 - - openjdk==11.0.15 - - openai==1.13.3 + - openjdk>=11.0.15, <12.0.0 + - openai>=1.13.3 - mkdocs-material==9.5.20 - mkdocs-material-extensions==1.3.1 - mkdocstrings==0.22.0 @@ -52,33 +51,34 @@ dependencies: - mkdocs-macros-plugin==1.0.1 - pygments==2.16.1 - pymdown-extensions==10.8.1 - - databricks-sql-connector==2.9.3 - - semver==3.0.0 - - xlrd==2.0.1 - - pygithub==1.59.0 - - pydantic==2.6.0 - - pyjwt==2.8.0 - - web3==6.16.0 + - databricks-sql-connector>=3.1.0 + - semver>=3.0.0 + - xlrd>=2.0.1 + - pygithub>=1.59.0 + - pydantic>=2.6.0 + - pyjwt>=2.8.0 - twine==4.0.2 - - delta-sharing-python==1.0.0 - - polars==0.18.8 - - moto[s3]==4.1.14 + - delta-sharing-python>=1.0.0 + - polars>=0.18.8 + - moto[s3]>=4.1.14 - xarray>=2023.1.0,<2023.8.0 - - ecmwf-api-client==1.6.3 - - netCDF4==1.6.4 - - black==24.1.0 + - ecmwf-api-client>=1.6.3 + - netCDF4>=1.6.4 + - black>=24.1.0 - joblib==1.3.2 - - great-expectations==0.18.8 + - great-expectations>=0.18.8 - pip: - - databricks-sdk==0.20.0 - - dependency-injector==4.41.0 - - azure-functions==1.15.0 - - azure-mgmt-eventgrid==10.2.0 - - hvac==1.1.1 - - langchain==0.1.17 + - databricks-sdk>=0.20.0 + - dependency-injector>=4.41.0 + - azure-functions>=1.15.0 + - azure-mgmt-eventgrid>=10.2.0 + - hvac>=1.1.1 + - langchain>=0.1.17 - build==0.10.0 - - deltalake==0.10.1 - - trio==0.22.1 - - sqlparams==5.1.0 - - entsoe-py==0.5.10 - - eth-typing>=4.2.2 \ No newline at end of file + - deltalake>=0.10.1 + - trio>=0.22.1 + - sqlparams>=5.1.0 + - entsoe-py>=0.5.10 + - web3>=6.18.0 + - eth-typing>=4.2.3 + - pandas>=1.5.2,<2.2.0 diff --git a/setup.py b/setup.py index 1df183744..537b5f671 100644 --- a/setup.py +++ b/setup.py @@ -29,19 +29,19 @@ long_description = (here / "PYPI-README.md").read_text() INSTALL_REQUIRES = [ - "databricks-sql-connector==3.1.0", - "azure-identity==1.12.0", - "pandas>=1.5.2,<3.0.0", - "jinja2==3.1.2", + "databricks-sql-connector>=3.1.0", + "azure-identity>=1.12.0", + "pandas>=1.5.2,<2.2.0", + "jinja2>=3.1.2", "importlib_metadata>=1.0.0", - "semver==3.0.0", - "xlrd==2.0.1", + "semver>=3.0.0", + "xlrd>=2.0.1", "grpcio>=1.48.1", "grpcio-status>=1.48.1", "googleapis-common-protos>=1.56.4", - "langchain==0.1.17", - "openai==1.13.3", - "pydantic==2.6.0", + "langchain>=0.1.17", + "openai>=1.13.3", + "pydantic>=2.6.0", ] PYSPARK_PACKAGES = [ @@ -50,24 +50,23 @@ ] PIPELINE_PACKAGES = [ - "dependency-injector==4.41.0", - "databricks-sdk==0.20.0", - "azure-storage-file-datalake==12.12.0", - "azure-mgmt-storage==21.0.0", - "azure-mgmt-eventgrid==10.2.0", - "boto3==1.28.2", - "hvac==1.1.1", - "azure-keyvault-secrets==4.7.0", - "web3==6.16.0", - "eth-typing>=4.2.2", - "polars[deltalake]==0.18.8", - "delta-sharing==1.0.0", + "dependency-injector>=4.41.0", + "databricks-sdk>=0.20.0", + "azure-storage-file-datalake>=12.12.0", + "azure-mgmt-storage>=21.0.0", + "azure-mgmt-eventgrid>=10.2.0", + "boto3>=1.28.2", + "hvac>=1.1.1", + "azure-keyvault-secrets>=4.7.0", + "web3>=6.18.0", + "polars[deltalake]>=0.18.8", + "delta-sharing>=1.0.0", "xarray>=2023.1.0,<2023.8.0", - "ecmwf-api-client==1.6.3", - "netCDF4==1.6.4", - "joblib==1.3.2", - "sqlparams==5.1.0", - "entsoe-py==0.5.10", + "ecmwf-api-client>=1.6.3", + "netCDF4>=1.6.4", + "joblib>=1.3.2", + "sqlparams>=5.1.0", + "entsoe-py>=0.5.10", ] EXTRAS_DEPENDENCIES: dict[str, list[str]] = { diff --git a/tests/api/v1/test_api_circular_average.py b/tests/api/v1/test_api_circular_average.py index 0d4fccf63..a70869c8b 100644 --- a/tests/api/v1/test_api_circular_average.py +++ b/tests/api/v1/test_api_circular_average.py @@ -73,7 +73,7 @@ async def test_api_circular_average_get_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) @@ -138,7 +138,7 @@ async def test_api_circular_average_post_validation_error(mocker: MockerFixture) assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) diff --git a/tests/api/v1/test_api_circular_standard_deviation.py b/tests/api/v1/test_api_circular_standard_deviation.py index a2ebac4ff..6239620cf 100644 --- a/tests/api/v1/test_api_circular_standard_deviation.py +++ b/tests/api/v1/test_api_circular_standard_deviation.py @@ -77,7 +77,7 @@ async def test_api_circular_standard_deviation_get_validation_error( assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) @@ -144,7 +144,7 @@ async def test_api_circular_standard_deviation_post_validation_error( assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) diff --git a/tests/api/v1/test_api_interpolate.py b/tests/api/v1/test_api_interpolate.py index d08191f1b..850a7b80f 100644 --- a/tests/api/v1/test_api_interpolate.py +++ b/tests/api/v1/test_api_interpolate.py @@ -73,7 +73,7 @@ async def test_api_interpolate_get_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) @@ -138,7 +138,7 @@ async def test_api_interpolate_post_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) diff --git a/tests/api/v1/test_api_latest.py b/tests/api/v1/test_api_latest.py index f5b30ec23..e591676cc 100644 --- a/tests/api/v1/test_api_latest.py +++ b/tests/api/v1/test_api_latest.py @@ -157,7 +157,7 @@ async def test_api_latest_get_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null}]}' ) @@ -251,7 +251,7 @@ async def test_api_latest_post_no_tags_provided_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["body"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["body"],"msg":"Field required","input":null}]}' ) @@ -283,7 +283,7 @@ async def test_api_latest_post_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null}]}' ) diff --git a/tests/api/v1/test_api_metadata.py b/tests/api/v1/test_api_metadata.py index 653ff7223..577d65095 100644 --- a/tests/api/v1/test_api_metadata.py +++ b/tests/api/v1/test_api_metadata.py @@ -88,7 +88,7 @@ async def test_api_metadata_get_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null}]}' ) @@ -143,7 +143,7 @@ async def test_api_metadata_post_no_tags_provided_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["body"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["body"],"msg":"Field required","input":null}]}' ) @@ -162,7 +162,7 @@ async def test_api_metadata_post_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null}]}' ) diff --git a/tests/api/v1/test_api_plot.py b/tests/api/v1/test_api_plot.py index 717316227..ea2f9b7a6 100644 --- a/tests/api/v1/test_api_plot.py +++ b/tests/api/v1/test_api_plot.py @@ -89,7 +89,7 @@ async def test_api_plot_get_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) @@ -179,7 +179,7 @@ async def test_api_plot_post_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) diff --git a/tests/api/v1/test_api_raw.py b/tests/api/v1/test_api_raw.py index 674e93c20..51edebaec 100644 --- a/tests/api/v1/test_api_raw.py +++ b/tests/api/v1/test_api_raw.py @@ -84,7 +84,7 @@ async def test_api_raw_get_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) @@ -162,7 +162,7 @@ async def test_api_raw_post_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) diff --git a/tests/api/v1/test_api_resample.py b/tests/api/v1/test_api_resample.py index aa428c73e..f5fafbc39 100644 --- a/tests/api/v1/test_api_resample.py +++ b/tests/api/v1/test_api_resample.py @@ -71,7 +71,7 @@ async def test_api_resample_get_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) @@ -134,7 +134,7 @@ async def test_api_resample_post_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) diff --git a/tests/api/v1/test_api_sql.py b/tests/api/v1/test_api_sql.py index 2b52d9f7b..685136773 100644 --- a/tests/api/v1/test_api_sql.py +++ b/tests/api/v1/test_api_sql.py @@ -85,7 +85,7 @@ async def test_api_raw_post_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["body","sql_statement"],"msg":"Field required","input":{},"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["body","sql_statement"],"msg":"Field required","input":{}}]}' ) diff --git a/tests/api/v1/test_api_summary.py b/tests/api/v1/test_api_summary.py index 5b44b31bf..a731b31bf 100644 --- a/tests/api/v1/test_api_summary.py +++ b/tests/api/v1/test_api_summary.py @@ -78,7 +78,7 @@ async def test_api_summary_get_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) @@ -132,7 +132,7 @@ async def test_api_summary_post_validation_error(mocker: MockerFixture): assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) diff --git a/tests/api/v1/test_api_time_weighted_average.py b/tests/api/v1/test_api_time_weighted_average.py index d9b1f40ab..01a9a6e8c 100644 --- a/tests/api/v1/test_api_time_weighted_average.py +++ b/tests/api/v1/test_api_time_weighted_average.py @@ -75,7 +75,7 @@ async def test_api_time_weighted_average_get_validation_error(mocker: MockerFixt assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) @@ -142,7 +142,7 @@ async def test_api_time_weighted_average_post_validation_error(mocker: MockerFix assert response.status_code == 422 assert ( actual - == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}' ) diff --git a/tests/docs/test_macros.py b/tests/docs/test_macros.py index 5f07c8686..d3fd7a51c 100644 --- a/tests/docs/test_macros.py +++ b/tests/docs/test_macros.py @@ -15,6 +15,7 @@ import sys sys.path.insert(0, ".") +import os from datetime import datetime from pytest_mock import MockerFixture from docs.macros import define_env @@ -52,6 +53,7 @@ def render_markdown(self, text, context=None): def test_github_releases(mocker: MockerFixture): + os.environ["GITHUB_JOB"] = "job_deploy_mkdocs_github_pages" mocker.patch("docs.macros.Github", return_value=MockGithub()) config = load_config() env = MacrosPlugin() diff --git a/tests/sdk/python/rtdip_sdk/pipelines/deploy/test_databricks_deploy.py b/tests/sdk/python/rtdip_sdk/pipelines/deploy/test_databricks_deploy.py index c5a7a4af2..d20dc35f8 100644 --- a/tests/sdk/python/rtdip_sdk/pipelines/deploy/test_databricks_deploy.py +++ b/tests/sdk/python/rtdip_sdk/pipelines/deploy/test_databricks_deploy.py @@ -23,11 +23,8 @@ CreateJob, JobCluster, ClusterSpec, - JobCompute, - ComputeSpec, Task, NotebookTask, - ComputeSpecKind, AutoScale, RuntimeEngine, DataSecurityMode, diff --git a/tests/sdk/python/rtdip_sdk/pipelines/utilities/aws/test_s3_copy_utility.py b/tests/sdk/python/rtdip_sdk/pipelines/utilities/aws/test_s3_copy_utility.py index cf293f20d..c733542eb 100644 --- a/tests/sdk/python/rtdip_sdk/pipelines/utilities/aws/test_s3_copy_utility.py +++ b/tests/sdk/python/rtdip_sdk/pipelines/utilities/aws/test_s3_copy_utility.py @@ -19,7 +19,7 @@ import sys from datetime import datetime import boto3 -from moto import mock_s3 +from moto import mock_aws sys.path.insert(0, ".") @@ -31,7 +31,7 @@ from src.sdk.python.rtdip_sdk.data_models.storage_objects import storage_objects_utils -@mock_s3 +@mock_aws def test_s3_copy_utility(): length: int = 1024 random.seed(datetime.now().timestamp())