Merge pull request #741 from rtdip/develop

v0.11.4

docs/macros.py

@@ -11,14 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import os
 from github import Github
 
 
 def define_env(env):
     @env.macro
     def github_releases(owner, repo):
-        github_client = Github()
+        # due to rate limits, only get this data on release
+        release_env = os.environ.get("GITHUB_JOB", "dev")
+        if release_env != "job_deploy_mkdocs_github_pages":
+            return "----\r\n"
+
+        github_client = Github(retry=0, timeout=5)
         repo = github_client.get_repo("{}/{}".format(owner, repo))
         output = "----\r\n"
         for release in repo.get_releases():

environment.yml

@@ -19,66 +19,66 @@ channels:
     - defaults
 dependencies:
     - python>=3.9,<3.12
-    - jinja2==3.1.3
+    - jinja2>=3.1.3
     - pytest==7.4.0
     - pytest-mock==3.11.1
     - pytest-cov==4.1.0
     - pylint==2.17.4
-    - pip==23.1.2
+    - pip>=23.1.2
     - turbodbc==4.11.0
     - numpy>=1.23.4
-    - pandas>=1.5.2,<3.0.0
     - oauthlib>=3.2.2
     - cryptography>=38.0.3
-    - azure-identity==1.12.0
-    - azure-storage-file-datalake==12.12.0
-    - azure-keyvault-secrets==4.7.0
-    - azure-mgmt-storage==21.0.0
-    - boto3==1.28.2
-    - pyodbc==4.0.39
-    - fastapi==0.110.0
-    - httpx==0.24.1
+    - azure-identity>=1.12.0
+    - azure-storage-file-datalake>=12.12.0
+    - azure-keyvault-secrets>=4.7.0
+    - azure-mgmt-storage>=21.0.0
+    - boto3>=1.28.2
+    - pyodbc>=4.0.39
+    - fastapi>=0.110.0
+    - httpx>=0.24.1
     - pyspark>=3.3.0,<3.6.0
     - delta-spark>=2.2.0,<3.3.0
     - grpcio>=1.48.1
     - grpcio-status>=1.48.1
     - googleapis-common-protos>=1.56.4
-    - openjdk==11.0.15
-    - openai==1.13.3
+    - openjdk>=11.0.15, <12.0.0
+    - openai>=1.13.3
     - mkdocs-material==9.5.20
     - mkdocs-material-extensions==1.3.1
     - mkdocstrings==0.22.0
     - mkdocstrings-python==1.4.0
     - mkdocs-macros-plugin==1.0.1
     - pygments==2.16.1
     - pymdown-extensions==10.8.1
-    - databricks-sql-connector==2.9.3
-    - semver==3.0.0
-    - xlrd==2.0.1
-    - pygithub==1.59.0
-    - pydantic==2.6.0
-    - pyjwt==2.8.0
-    - web3==6.16.0
+    - databricks-sql-connector>=3.1.0
+    - semver>=3.0.0
+    - xlrd>=2.0.1
+    - pygithub>=1.59.0
+    - pydantic>=2.6.0
+    - pyjwt>=2.8.0
     - twine==4.0.2
-    - delta-sharing-python==1.0.0
-    - polars==0.18.8
-    - moto[s3]==4.1.14
+    - delta-sharing-python>=1.0.0
+    - polars>=0.18.8
+    - moto[s3]>=4.1.14
     - xarray>=2023.1.0,<2023.8.0
-    - ecmwf-api-client==1.6.3
-    - netCDF4==1.6.4
-    - black==24.1.0
+    - ecmwf-api-client>=1.6.3
+    - netCDF4>=1.6.4
+    - black>=24.1.0
     - joblib==1.3.2
-    - great-expectations==0.18.8
+    - great-expectations>=0.18.8
     - pip:
-        - databricks-sdk==0.20.0    
-        - dependency-injector==4.41.0
-        - azure-functions==1.15.0
-        - azure-mgmt-eventgrid==10.2.0
-        - hvac==1.1.1
-        - langchain==0.1.17
+        - databricks-sdk>=0.20.0    
+        - dependency-injector>=4.41.0
+        - azure-functions>=1.15.0
+        - azure-mgmt-eventgrid>=10.2.0
+        - hvac>=1.1.1
+        - langchain>=0.1.17
         - build==0.10.0
-        - deltalake==0.10.1
-        - trio==0.22.1
-        - sqlparams==5.1.0
-        - entsoe-py==0.5.10
-        - eth-typing>=4.2.2
+        - deltalake>=0.10.1
+        - trio>=0.22.1
+        - sqlparams>=5.1.0
+        - entsoe-py>=0.5.10
+        - web3>=6.18.0
+        - eth-typing>=4.2.3
+        - pandas>=1.5.2,<2.2.0

setup.py

@@ -29,19 +29,19 @@
 long_description = (here / "PYPI-README.md").read_text()
 
 INSTALL_REQUIRES = [
-    "databricks-sql-connector==3.1.0",
-    "azure-identity==1.12.0",
-    "pandas>=1.5.2,<3.0.0",
-    "jinja2==3.1.2",
+    "databricks-sql-connector>=3.1.0",
+    "azure-identity>=1.12.0",
+    "pandas>=1.5.2,<2.2.0",
+    "jinja2>=3.1.2",
     "importlib_metadata>=1.0.0",
-    "semver==3.0.0",
-    "xlrd==2.0.1",
+    "semver>=3.0.0",
+    "xlrd>=2.0.1",
     "grpcio>=1.48.1",
     "grpcio-status>=1.48.1",
     "googleapis-common-protos>=1.56.4",
-    "langchain==0.1.17",
-    "openai==1.13.3",
-    "pydantic==2.6.0",
+    "langchain>=0.1.17",
+    "openai>=1.13.3",
+    "pydantic>=2.6.0",
 ]
 
 PYSPARK_PACKAGES = [
@@ -50,24 +50,23 @@
 ]
 
 PIPELINE_PACKAGES = [
-    "dependency-injector==4.41.0",
-    "databricks-sdk==0.20.0",
-    "azure-storage-file-datalake==12.12.0",
-    "azure-mgmt-storage==21.0.0",
-    "azure-mgmt-eventgrid==10.2.0",
-    "boto3==1.28.2",
-    "hvac==1.1.1",
-    "azure-keyvault-secrets==4.7.0",
-    "web3==6.16.0",
-    "eth-typing>=4.2.2",
-    "polars[deltalake]==0.18.8",
-    "delta-sharing==1.0.0",
+    "dependency-injector>=4.41.0",
+    "databricks-sdk>=0.20.0",
+    "azure-storage-file-datalake>=12.12.0",
+    "azure-mgmt-storage>=21.0.0",
+    "azure-mgmt-eventgrid>=10.2.0",
+    "boto3>=1.28.2",
+    "hvac>=1.1.1",
+    "azure-keyvault-secrets>=4.7.0",
+    "web3>=6.18.0",
+    "polars[deltalake]>=0.18.8",
+    "delta-sharing>=1.0.0",
     "xarray>=2023.1.0,<2023.8.0",
-    "ecmwf-api-client==1.6.3",
-    "netCDF4==1.6.4",
-    "joblib==1.3.2",
-    "sqlparams==5.1.0",
-    "entsoe-py==0.5.10",
+    "ecmwf-api-client>=1.6.3",
+    "netCDF4>=1.6.4",
+    "joblib>=1.3.2",
+    "sqlparams>=5.1.0",
+    "entsoe-py>=0.5.10",
 ]
 
 EXTRAS_DEPENDENCIES: dict[str, list[str]] = {

tests/api/v1/test_api_circular_average.py

@@ -73,7 +73,7 @@ async def test_api_circular_average_get_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -138,7 +138,7 @@ async def test_api_circular_average_post_validation_error(mocker: MockerFixture)
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 

tests/api/v1/test_api_circular_standard_deviation.py

@@ -77,7 +77,7 @@ async def test_api_circular_standard_deviation_get_validation_error(
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -144,7 +144,7 @@ async def test_api_circular_standard_deviation_post_validation_error(
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 

tests/api/v1/test_api_interpolate.py

@@ -73,7 +73,7 @@ async def test_api_interpolate_get_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -138,7 +138,7 @@ async def test_api_interpolate_post_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 

tests/api/v1/test_api_latest.py

@@ -157,7 +157,7 @@ async def test_api_latest_get_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -251,7 +251,7 @@ async def test_api_latest_post_no_tags_provided_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["body"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["body"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -283,7 +283,7 @@ async def test_api_latest_post_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null}]}'
     )
 
 

tests/api/v1/test_api_metadata.py

@@ -88,7 +88,7 @@ async def test_api_metadata_get_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -143,7 +143,7 @@ async def test_api_metadata_post_no_tags_provided_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["body"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["body"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -162,7 +162,7 @@ async def test_api_metadata_post_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","business_unit"],"msg":"Field required","input":null}]}'
     )
 
 

tests/api/v1/test_api_plot.py

@@ -89,7 +89,7 @@ async def test_api_plot_get_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -179,7 +179,7 @@ async def test_api_plot_post_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 

tests/api/v1/test_api_raw.py

@@ -84,7 +84,7 @@ async def test_api_raw_get_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -162,7 +162,7 @@ async def test_api_raw_post_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 

tests/api/v1/test_api_resample.py

@@ -71,7 +71,7 @@ async def test_api_resample_get_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -134,7 +134,7 @@ async def test_api_resample_post_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 

tests/api/v1/test_api_sql.py

@@ -85,7 +85,7 @@ async def test_api_raw_post_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["body","sql_statement"],"msg":"Field required","input":{},"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["body","sql_statement"],"msg":"Field required","input":{}}]}'
     )
 
 

tests/api/v1/test_api_summary.py

@@ -78,7 +78,7 @@ async def test_api_summary_get_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )
 
 
@@ -132,7 +132,7 @@ async def test_api_summary_post_validation_error(mocker: MockerFixture):
     assert response.status_code == 422
     assert (
         actual
-        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}'
+        == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null}]}'
     )