diff --git a/pyright/alt-1/requirements-pinned.txt b/pyright/alt-1/requirements-pinned.txt index d940cdc441808..88cb07accb3ea 100644 --- a/pyright/alt-1/requirements-pinned.txt +++ b/pyright/alt-1/requirements-pinned.txt @@ -7,7 +7,8 @@ aiosignal==1.3.1 alembic==1.13.1 aniso8601==9.0.1 annotated-types==0.6.0 -anyio==4.2.0 +anyio==4.3.0 +appnope==0.1.4 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 arrow==1.3.0 @@ -36,7 +37,7 @@ colored==1.4.4 coloredlogs==14.0 comm==0.2.1 contourpy==1.2.0 -coverage==7.4.1 +coverage==7.4.2 croniter==2.0.1 cryptography==41.0.7 cycler==0.12.1 @@ -102,7 +103,6 @@ gql==3.5.0 graphene==3.3 graphql-core==3.2.3 graphql-relay==3.2.0 -greenlet==3.0.3 grpcio==1.60.1 grpcio-health-checking==1.60.1 grpcio-tools==1.60.1 @@ -112,7 +112,7 @@ httplib2==0.22.0 httptools==0.6.1 httpx==0.26.0 humanfriendly==10.0 -hypothesis==6.98.8 +hypothesis==6.98.9 idna==3.6 importlib-metadata==6.11.0 iniconfig==2.0.0 @@ -123,11 +123,10 @@ isoduration==20.11.0 isort==5.13.2 jaraco.classes==3.3.1 jedi==0.19.1 -jeepney==0.8.0 Jinja2==3.1.3 jmespath==1.0.1 joblib==1.3.2 -json5==0.9.14 +json5==0.9.17 jsonpointer==2.4 jsonschema==4.21.1 jsonschema-specifications==2023.12.1 @@ -137,7 +136,7 @@ jupyter_client==8.6.0 jupyter_core==5.7.1 jupyter_server==2.12.5 jupyter_server_terminals==0.5.2 -jupyterlab==4.1.1 +jupyterlab==4.1.2 jupyterlab_pygments==0.3.0 jupyterlab_server==2.25.3 keyring==24.3.0 @@ -160,12 +159,12 @@ more-itertools==10.2.0 morefs==0.2.0 msgpack==1.0.7 multidict==6.0.5 -multimethod==1.11 +multimethod==1.11.1 mypy==1.8.0 mypy-extensions==1.0.0 mypy-protobuf==3.5.0 nbclient==0.9.0 -nbconvert==7.16.0 +nbconvert==7.16.1 nbformat==5.9.2 nest-asyncio==1.6.0 networkx==3.2.1 @@ -191,7 +190,7 @@ pexpect==4.9.0 pillow==10.2.0 platformdirs==3.11.0 pluggy==1.4.0 -polars==0.20.9 +polars==0.20.10 -e examples/project_fully_featured prometheus_client==0.20.0 prompt-toolkit==3.0.43 @@ -247,7 +246,6 @@ s3transfer==0.10.0 scikit-learn==1.4.1.post1 scipy==1.12.0 seaborn==0.13.2 -SecretStorage==3.3.3 Send2Trash==1.8.2 six==1.16.0 slack_sdk==3.27.0 diff --git a/pyright/master/requirements-pinned.txt b/pyright/master/requirements-pinned.txt index 46f5cbca9cd8a..4ea60c81c43e1 100644 --- a/pyright/master/requirements-pinned.txt +++ b/pyright/master/requirements-pinned.txt @@ -10,10 +10,10 @@ alembic==1.13.1 altair==4.2.2 amqp==5.2.0 aniso8601==9.0.1 -anyio==4.2.0 +anyio==4.3.0 apache-airflow==2.7.3 apache-airflow-providers-apache-spark==4.7.1 -apache-airflow-providers-cncf-kubernetes==7.14.0 +apache-airflow-providers-cncf-kubernetes==8.0.0 apache-airflow-providers-common-sql==1.11.0 apache-airflow-providers-docker==3.9.1 apache-airflow-providers-ftp==3.7.0 @@ -24,6 +24,7 @@ apeye==1.4.1 apeye-core==1.1.5 apispec==6.4.0 appdirs==1.4.4 +appnope==0.1.4 argcomplete==3.2.2 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 @@ -37,7 +38,7 @@ async-lru==2.0.4 async-timeout==4.0.3 attrs==23.2.0 autodocsumm==0.2.12 -autoflake==2.2.1 +autoflake==2.3.0 -e python_modules/automation avro==1.11.3 avro-gen3==0.7.11 @@ -56,8 +57,8 @@ bitmath==1.3.3.1 bleach==6.1.0 blinker==1.7.0 bokeh==3.3.4 -boto3==1.34.44 -botocore==1.34.44 +boto3==1.34.46 +botocore==1.34.46 buildkite-test-collector==0.1.7 CacheControl==0.14.0 cached-property==1.5.2 @@ -89,7 +90,7 @@ ConfigUpdater==3.2 confluent-kafka==2.3.0 connexion==2.14.2 contourpy==1.2.0 -coverage==7.4.1 +coverage==7.4.2 cron-descriptor==1.4.3 croniter==2.0.1 cryptography==41.0.7 @@ -133,6 +134,7 @@ cycler==0.12.1 -e python_modules/libraries/dagster-mlflow -e python_modules/libraries/dagster-msteams -e python_modules/libraries/dagster-mysql +-e python_modules/libraries/dagster-openai -e python_modules/libraries/dagster-pagerduty -e python_modules/libraries/dagster-pandas -e python_modules/libraries/dagster-pandera @@ -182,6 +184,7 @@ diff-match-patch==20200713 dill==0.3.8 distlib==0.3.8 distributed==2024.2.0 +distro==1.9.0 dnspython==2.6.1 docker==5.0.3 docker-image-py==0.1.12 @@ -240,7 +243,6 @@ graphql-core==3.2.3 graphql-relay==3.2.0 graphviz==0.20.1 great-expectations==0.17.11 -greenlet==3.0.3 grpcio==1.60.1 grpcio-health-checking==1.60.1 grpcio-status==1.60.1 @@ -254,7 +256,7 @@ httplib2==0.22.0 httptools==0.6.1 httpx==0.26.0 humanfriendly==10.0 -hypothesis==6.98.8 +hypothesis==6.98.9 idna==3.6 ijson==3.2.3 imagesize==1.4.1 @@ -275,10 +277,10 @@ Jinja2==3.1.2 jmespath==1.0.1 joblib==1.3.2 jschema-to-python==1.2.3 -json5==0.9.14 +json5==0.9.17 jsondiff==2.0.0 jsonpatch==1.33 -jsonpickle==3.0.2 +jsonpickle==3.0.3 jsonpointer==2.4 jsonref==1.1.0 jsonschema==4.21.1 @@ -291,7 +293,7 @@ jupyter_client==7.4.9 jupyter_core==5.7.1 jupyter_server==2.12.5 jupyter_server_terminals==0.5.2 -jupyterlab==4.1.1 +jupyterlab==4.1.2 jupyterlab_pygments==0.3.0 jupyterlab_server==2.25.3 jupyterlab_widgets==3.0.10 @@ -299,8 +301,8 @@ jwt==1.3.1 kiwisolver==1.4.5 kombu==5.3.5 kopf==1.37.1 -kubernetes==23.6.0 -kubernetes-asyncio==24.2.3 +kubernetes==29.0.0 +kubernetes_asyncio==29.0.0 lazy-object-proxy==1.10.0 leather==0.3.4 limits==3.9.0 @@ -335,13 +337,13 @@ msal==1.26.0 msal-extensions==1.1.0 msgpack==1.0.7 multidict==6.0.5 -multimethod==1.11 +multimethod==1.11.1 mypy-extensions==1.0.0 mypy-protobuf==3.5.0 mysql-connector-python==8.3.0 natsort==8.4.0 nbclient==0.9.0 -nbconvert==7.16.0 +nbconvert==7.16.1 nbformat==5.9.2 nest-asyncio==1.6.0 networkx==2.8.8 @@ -351,24 +353,13 @@ noteable-origami==0.0.35 notebook==7.1.0 notebook_shim==0.2.4 numpy==1.26.4 -nvidia-cublas-cu12==12.1.3.1 -nvidia-cuda-cupti-cu12==12.1.105 -nvidia-cuda-nvrtc-cu12==12.1.105 -nvidia-cuda-runtime-cu12==12.1.105 -nvidia-cudnn-cu12==8.9.2.26 -nvidia-cufft-cu12==11.0.2.54 -nvidia-curand-cu12==10.3.2.106 -nvidia-cusolver-cu12==11.4.5.107 -nvidia-cusparse-cu12==12.1.0.106 -nvidia-nccl-cu12==2.19.3 -nvidia-nvjitlink-cu12==12.3.101 -nvidia-nvtx-cu12==12.1.105 oauth2client==4.1.3 oauthlib==3.2.2 objgraph==3.6.0 onnx==1.15.0 onnxconverter-common==1.13.0 onnxruntime==1.17.0 +openai==1.12.0 openapi-schema-validator==0.6.2 openapi-spec-validator==0.7.1 opentelemetry-api==1.22.0 @@ -408,7 +399,7 @@ pkginfo==1.9.6 platformdirs==3.11.0 plotly==5.19.0 pluggy==1.4.0 -polars==0.20.9 +polars==0.20.10 portalocker==2.8.2 prison==0.2.1 progressbar2==4.3.2 @@ -486,17 +477,17 @@ scikit-learn==1.4.1.post1 scipy==1.12.0 scrapbook==0.5.0 seaborn==0.13.2 -selenium==4.17.2 +selenium==4.18.1 Send2Trash==1.8.2 sending==0.3.0 -sentry-sdk==1.40.4 +sentry-sdk==1.40.5 setproctitle==1.3.3 six==1.16.0 skein==0.8.2 skl2onnx==1.16.0 slack_sdk==3.27.0 -sling==1.1.5.post4 -sling-linux-amd64==1.1.5.post4 +sling==1.1.6.post1 +sling-mac-universal==1.1.6.post1 smmap==5.0.1 sniffio==1.3.0 snowballstemmer==2.2.0 @@ -549,7 +540,6 @@ tqdm==4.66.2 traitlets==5.14.1 trio==0.24.0 trio-websocket==0.11.1 -triton==2.2.0 -e examples/tutorial_notebook_assets twilio==8.13.0 twine==1.15.0 diff --git a/pyright/master/requirements.txt b/pyright/master/requirements.txt index f34ff302f08db..81e57da2c6339 100644 --- a/pyright/master/requirements.txt +++ b/pyright/master/requirements.txt @@ -68,6 +68,7 @@ -e python_modules/libraries/dagster-mlflow/ -e python_modules/libraries/dagster-msteams/ -e python_modules/libraries/dagster-mysql/ +-e python_modules/libraries/dagster-openai/ -e python_modules/libraries/dagster-pagerduty/ -e python_modules/libraries/dagster-pandas/ -e python_modules/libraries/dagster-pandera/ diff --git a/python_modules/libraries/dagster-openai/.coveragerc b/python_modules/libraries/dagster-openai/.coveragerc new file mode 100644 index 0000000000000..398ff08afa472 --- /dev/null +++ b/python_modules/libraries/dagster-openai/.coveragerc @@ -0,0 +1,2 @@ +[run] +branch = True diff --git a/python_modules/libraries/dagster-openai/LICENSE b/python_modules/libraries/dagster-openai/LICENSE new file mode 100644 index 0000000000000..3b54f435cb047 --- /dev/null +++ b/python_modules/libraries/dagster-openai/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024 Dagster Labs, Inc". + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/python_modules/libraries/dagster-openai/MANIFEST.in b/python_modules/libraries/dagster-openai/MANIFEST.in new file mode 100644 index 0000000000000..8e5c16d383f8b --- /dev/null +++ b/python_modules/libraries/dagster-openai/MANIFEST.in @@ -0,0 +1,3 @@ +include README.md +include LICENSE +include dagster_openai/py.typed diff --git a/python_modules/libraries/dagster-openai/README.md b/python_modules/libraries/dagster-openai/README.md new file mode 100644 index 0000000000000..42af42c36bbd8 --- /dev/null +++ b/python_modules/libraries/dagster-openai/README.md @@ -0,0 +1 @@ +# dagster-openai diff --git a/python_modules/libraries/dagster-openai/dagster_openai/__init__.py b/python_modules/libraries/dagster-openai/dagster_openai/__init__.py new file mode 100644 index 0000000000000..551edf064a841 --- /dev/null +++ b/python_modules/libraries/dagster-openai/dagster_openai/__init__.py @@ -0,0 +1,10 @@ +from dagster._core.libraries import DagsterLibraryRegistry + +from .resources import ( + OpenAIResource as OpenAIResource, + with_usage_metadata as with_usage_metadata, +) + +# TODO: replace version by `__version__` when we add back a version.py file and publish library to pypi. +# Import with `from .version import __version__` +DagsterLibraryRegistry.register("dagster-openai", "1!0+dev") diff --git a/python_modules/libraries/dagster-openai/dagster_openai/py.typed b/python_modules/libraries/dagster-openai/dagster_openai/py.typed new file mode 100644 index 0000000000000..b648ac9233330 --- /dev/null +++ b/python_modules/libraries/dagster-openai/dagster_openai/py.typed @@ -0,0 +1 @@ +partial diff --git a/python_modules/libraries/dagster-openai/dagster_openai/resources.py b/python_modules/libraries/dagster-openai/dagster_openai/resources.py new file mode 100644 index 0000000000000..518e9a8530469 --- /dev/null +++ b/python_modules/libraries/dagster-openai/dagster_openai/resources.py @@ -0,0 +1,382 @@ +from collections import defaultdict +from contextlib import contextmanager +from enum import Enum +from functools import wraps +from typing import Generator, Optional, Union +from weakref import WeakKeyDictionary + +from dagster import ( + AssetExecutionContext, + AssetKey, + ConfigurableResource, + InitResourceContext, + OpExecutionContext, +) +from dagster._annotations import experimental +from dagster._core.errors import ( + DagsterInvariantViolationError, +) +from openai import Client +from pydantic import Field, PrivateAttr + + +class ApiEndpointClassesEnum(Enum): + """Supported endpoint classes of the OpenAI API v1.""" + + COMPLETIONS = "completions" + CHAT = "chat" + EMBEDDINGS = "embeddings" + + +API_ENDPOINT_CLASSES_TO_ENDPOINT_METHODS_MAPPING = { + ApiEndpointClassesEnum.COMPLETIONS: [["create"]], + ApiEndpointClassesEnum.CHAT: [["completions", "create"]], + ApiEndpointClassesEnum.EMBEDDINGS: [["create"]], +} + +context_to_counters = WeakKeyDictionary() + + +def _add_to_asset_metadata( + context: AssetExecutionContext, usage_metadata: dict, output_name: Optional[str] +): + if context not in context_to_counters: + context_to_counters[context] = defaultdict(lambda: 0) + counters = context_to_counters[context] + + for metadata_key, delta in usage_metadata.items(): + counters[metadata_key] += delta + context.add_output_metadata(dict(counters), output_name) + + +@experimental +def with_usage_metadata(context: AssetExecutionContext, output_name: Optional[str], func): + """This wrapper can be used on any endpoint of the + `openai library ` + to log the OpenAI API usage metadata in the asset metadata. + + Examples: + .. code-block:: python + + from dagster import ( + AssetExecutionContext, + AssetKey, + AssetSelection, + AssetSpec, + Definitions, + EnvVar, + MaterializeResult, + asset, + define_asset_job, + multi_asset, + ) + from dagster_openai import OpenAIResource, with_usage_metadata + + + @asset(compute_kind="OpenAI") + def openai_asset(context: AssetExecutionContext, openai: OpenAIResource): + with openai.get_client(context) as client: + client.fine_tuning.jobs.create = with_usage_metadata( + context=context, output_name="some_output_name", func=client.fine_tuning.jobs.create + ) + client.fine_tuning.jobs.create(model="gpt-3.5-turbo", training_file="some_training_file") + + + openai_asset_job = define_asset_job(name="openai_asset_job", selection="openai_asset") + + + @multi_asset( + specs=[ + AssetSpec("my_asset1"), + AssetSpec("my_asset2"), + ] + ) + def openai_multi_asset(context: AssetExecutionContext, openai: OpenAIResource): + with openai.get_client(context, asset_key=AssetKey("my_asset1")) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Say this is a test"}] + ) + + # The materialization of `my_asset1` will include both OpenAI usage metadata + # and the metadata added when calling `MaterializeResult`. + return ( + MaterializeResult(asset_key="my_asset1", metadata={"foo": "bar"}), + MaterializeResult(asset_key="my_asset2", metadata={"baz": "qux"}), + ) + + + openai_multi_asset_job = define_asset_job( + name="openai_multi_asset_job", selection=AssetSelection.assets(openai_multi_asset) + ) + + + defs = Definitions( + assets=[openai_asset, openai_multi_asset], + jobs=[openai_asset_job, openai_multi_asset_job], + resources={ + "openai": OpenAIResource(api_key=EnvVar("OPENAI_API_KEY")), + }, + ) + """ + if not isinstance(context, AssetExecutionContext): + raise DagsterInvariantViolationError( + "The `with_usage_metadata` can only be used when context is of type `AssetExecutionContext`." + ) + + @wraps(func) + def wrapper(*args, **kwargs): + response = func(*args, **kwargs) + usage = response.usage + usage_metadata = { + "openai.calls": 1, + "openai.total_tokens": usage.total_tokens, + "openai.prompt_tokens": usage.prompt_tokens, + } + if hasattr(usage, "completion_tokens"): + usage_metadata["openai.completion_tokens"] = usage.completion_tokens + _add_to_asset_metadata(context, usage_metadata, output_name) + + return response + + return wrapper + + +@experimental +class OpenAIResource(ConfigurableResource): + """This resource is wrapper over the + `openai library `_. + + By configuring this OpenAI resource, you can interact with OpenAI API + and log its usage metadata in the asset metadata. + + Examples: + .. code-block:: python + + import os + + from dagster import AssetExecutionContext, Definitions, EnvVar, asset, define_asset_job + from dagster_openai import OpenAIResource + + + @asset(compute_kind="OpenAI") + def openai_asset(context: AssetExecutionContext, openai: OpenAIResource): + with openai.get_client(context) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Say this is a test"}] + ) + + openai_asset_job = define_asset_job(name="openai_asset_job", selection="openai_asset") + + defs = Definitions( + assets=[openai_asset], + jobs=[openai_asset_job], + resources={ + "openai": OpenAIResource(api_key=EnvVar("OPENAI_API_KEY")), + }, + ) + """ + + api_key: str = Field(description=("OpenAI API key. See https://platform.openai.com/api-keys")) + + _client: Client = PrivateAttr() + + @classmethod + def _is_dagster_maintained(cls) -> bool: + return True + + def _wrap_with_usage_metadata( + self, + api_endpoint_class: ApiEndpointClassesEnum, + context: AssetExecutionContext, + output_name: Optional[str], + ): + for attribute_names in API_ENDPOINT_CLASSES_TO_ENDPOINT_METHODS_MAPPING[api_endpoint_class]: + curr = self._client.__getattribute__(api_endpoint_class.value) + # Get the second to last attribute from the attribute list to reach the method. + i = 0 + while i < len(attribute_names) - 1: + curr = curr.__getattribute__(attribute_names[i]) + i += 1 + # Wrap the method. + curr.__setattr__( + attribute_names[i], + with_usage_metadata( + context=context, + output_name=output_name, + func=curr.__getattribute__(attribute_names[i]), + ), + ) + + def setup_for_execution(self, context: InitResourceContext) -> None: + # Set up an OpenAI client based on the API key. + self._client = Client(api_key=self.api_key) + + @contextmanager + def get_client( + self, context: Union[AssetExecutionContext, OpExecutionContext] + ) -> Generator[Client, None, None]: + """Yields an ``openai.Client`` for interacting with the OpenAI API. + + By default, in an asset context, the client comes with wrapped endpoints + for three API resources, Completions, Embeddings and Chat, + allowing to log the API usage metadata in the asset metadata. + + Note that the endpoints are not and cannot be wrapped + to automatically capture the API usage metadata in an op context. + + :param context: The ``context`` object for computing the op or asset in which ``get_client`` is called. + + Examples: + .. code-block:: python + + from dagster import ( + AssetExecutionContext, + Definitions, + EnvVar, + GraphDefinition, + OpExecutionContext, + asset, + define_asset_job, + op, + ) + from dagster_openai import OpenAIResource + + + @op + def openai_op(context: OpExecutionContext, openai: OpenAIResource): + with openai.get_client(context) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Say this is a test"}] + ) + + + openai_op_job = GraphDefinition(name="openai_op_job", node_defs=[openai_op]).to_job() + + + @asset(compute_kind="OpenAI") + def openai_asset(context: AssetExecutionContext, openai: OpenAIResource): + with openai.get_client(context) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Say this is a test"}] + ) + + + openai_asset_job = define_asset_job(name="openai_asset_job", selection="openai_asset") + + defs = Definitions( + assets=[openai_asset], + jobs=[openai_asset_job, openai_op_job], + resources={ + "openai": OpenAIResource(api_key=EnvVar("OPENAI_API_KEY")), + }, + ) + """ + yield from self._get_client(context=context, asset_key=None) + + @contextmanager + def get_client_for_asset( + self, context: AssetExecutionContext, asset_key: AssetKey + ) -> Generator[Client, None, None]: + """Yields an ``openai.Client`` for interacting with the OpenAI. + + When using this method, the OpenAI API usage metadata is automatically + logged in the asset materializations associated with the provided ``asset_key``. + + By default, the client comes with wrapped endpoints + for three API resources, Completions, Embeddings and Chat, + allowing to log the API usage metadata in the asset metadata. + + This method can only be called when working with assets, + i.e. the provided ``context`` must be of type ``AssetExecutionContext. + + :param context: The ``context`` object for computing the asset in which ``get_client`` is called. + :param asset_key: the ``asset_key`` of the asset for which a materialization should include the metadata. + + Examples: + .. code-block:: python + + from dagster import ( + AssetExecutionContext, + AssetKey, + AssetSpec, + Definitions, + EnvVar, + MaterializeResult, + asset, + define_asset_job, + multi_asset, + ) + from dagster_openai import OpenAIResource + + + @asset(compute_kind="OpenAI") + def openai_asset(context: AssetExecutionContext, openai: OpenAIResource): + with openai.get_client_for_asset(context, context.asset_key) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Say this is a test"}] + ) + + + openai_asset_job = define_asset_job(name="openai_asset_job", selection="openai_asset") + + + @multi_asset(specs=[AssetSpec("my_asset1"), AssetSpec("my_asset2")], compute_kind="OpenAI") + def openai_multi_asset(context: AssetExecutionContext, openai_resource: OpenAIResource): + with openai_resource.get_client_for_asset(context, asset_key=AssetKey("my_asset1")) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Say this is a test"}] + ) + return ( + MaterializeResult(asset_key="my_asset1", metadata={"some_key": "some_value1"}), + MaterializeResult(asset_key="my_asset2", metadata={"some_key": "some_value2"}), + ) + + + openai_multi_asset_job = define_asset_job( + name="openai_multi_asset_job", selection="openai_multi_asset" + ) + + defs = Definitions( + assets=[openai_asset, openai_multi_asset], + jobs=[openai_asset_job, openai_multi_asset_job], + resources={ + "openai": OpenAIResource(api_key=EnvVar("OPENAI_API_KEY")), + }, + ) + """ + yield from self._get_client(context=context, asset_key=asset_key) + + def _get_client( + self, + context: Union[AssetExecutionContext, OpExecutionContext], + asset_key: Optional[AssetKey] = None, + ) -> Generator[Client, None, None]: + if isinstance(context, AssetExecutionContext): + if asset_key is None: + if len(context.assets_def.keys_by_output_name.keys()) > 1: + raise DagsterInvariantViolationError( + "The argument `asset_key` must be specified for multi_asset with more than one asset." + ) + asset_key = context.asset_key + output_name = context.output_for_asset_key(asset_key) + # By default, when the resource is used in an asset context, + # we wrap the methods of `openai.resources.Completions`, + # `openai.resources.Embeddings` and `openai.resources.chat.Completions`. + # This allows the usage metadata to be captured in the asset metadata. + api_endpoint_classes = [ + ApiEndpointClassesEnum.COMPLETIONS, + ApiEndpointClassesEnum.CHAT, + ApiEndpointClassesEnum.EMBEDDINGS, + ] + for api_endpoint_class in api_endpoint_classes: + self._wrap_with_usage_metadata( + api_endpoint_class=api_endpoint_class, + context=context, + output_name=output_name, + ) + yield self._client + + def teardown_after_execution(self, context: InitResourceContext) -> None: + # Close OpenAI client. + self._client.close() diff --git a/python_modules/libraries/dagster-openai/dagster_openai_tests/__init__.py b/python_modules/libraries/dagster-openai/dagster_openai_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/libraries/dagster-openai/dagster_openai_tests/test_resources.py b/python_modules/libraries/dagster-openai/dagster_openai_tests/test_resources.py new file mode 100644 index 0000000000000..4f96599d4d3d0 --- /dev/null +++ b/python_modules/libraries/dagster-openai/dagster_openai_tests/test_resources.py @@ -0,0 +1,478 @@ +import pytest +from dagster import ( + AssetExecutionContext, + AssetKey, + AssetSelection, + AssetSpec, + Definitions, + OpExecutionContext, + StaticPartitionsDefinition, + asset, + define_asset_job, + graph_asset, + materialize_to_memory, + multi_asset, + op, +) +from dagster._core.errors import DagsterInvariantViolationError +from dagster._core.execution.context.init import build_init_resource_context +from dagster._utils.test import wrap_op_in_graph_and_execute +from dagster_openai import OpenAIResource, with_usage_metadata +from mock import ANY, MagicMock, patch + + +@patch("dagster_openai.resources.Client") +def test_openai_client(mock_client) -> None: + openai_resource = OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + openai_resource.setup_for_execution(build_init_resource_context()) + + mock_context = MagicMock() + with openai_resource.get_client(mock_context): + mock_client.assert_called_once_with(api_key="xoxp-1234123412341234-12341234-1234") + + +@patch("dagster_openai.resources.OpenAIResource._wrap_with_usage_metadata") +@patch("dagster.OpExecutionContext", autospec=OpExecutionContext) +@patch("dagster_openai.resources.Client") +def test_openai_resource_with_op(mock_client, mock_context, mock_wrapper): + @op + def openai_op(openai_resource: OpenAIResource): + assert openai_resource + + with openai_resource.get_client(context=mock_context) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Say this is a test"}] + ) + + assert mock_client.called + assert mock_wrapper.not_called + + result = wrap_op_in_graph_and_execute( + openai_op, + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + assert result.success + + +@patch("dagster_openai.resources.OpenAIResource._wrap_with_usage_metadata") +@patch("dagster.AssetExecutionContext", autospec=AssetExecutionContext) +@patch("dagster_openai.resources.Client") +def test_openai_resource_with_asset(mock_client, mock_context, mock_wrapper): + @asset + def openai_asset(openai_resource: OpenAIResource): + assert openai_resource + + with openai_resource.get_client(context=mock_context) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Say this is a test"}] + ) + + assert mock_client.called + assert mock_wrapper.call_count == 3 + + result = materialize_to_memory( + [openai_asset], + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + + assert result.success + + +@patch("dagster_openai.resources.OpenAIResource._wrap_with_usage_metadata") +@patch("dagster.AssetExecutionContext", autospec=AssetExecutionContext) +@patch("dagster_openai.resources.Client") +def test_openai_resource_with_graph_backed_asset(mock_client, mock_context, mock_wrapper): + @op + def model_version_op(): + return "gpt-3.5-turbo" + + @op + def message_op(): + return {"role": "user", "content": "Say this is a test"} + + @op + def openai_op(openai_resource: OpenAIResource, model_version, message): + assert openai_resource + + with openai_resource.get_client(context=mock_context) as client: + client.chat.completions.create(model=model_version, messages=[message]) + + assert mock_client.called + assert mock_wrapper.called + + @graph_asset + def openai_asset(): + return openai_op(model_version_op(), message_op()) + + result = materialize_to_memory( + [openai_asset], + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + + assert result.success + + +@patch("dagster_openai.resources.OpenAIResource._wrap_with_usage_metadata") +@patch("dagster.AssetExecutionContext", autospec=AssetExecutionContext) +@patch("dagster_openai.resources.Client") +def test_openai_resource_with_multi_asset(mock_client, mock_context, mock_wrapper): + @multi_asset( + specs=[AssetSpec("status"), AssetSpec("result")], + ) + def openai_multi_asset(openai_resource: OpenAIResource): + assert openai_resource + + mock_context.assets_def.keys_by_output_name.keys.return_value = [ + AssetKey("status"), + AssetKey("result"), + ] + mock_context.output_for_asset_key.return_value = "result" + + # Test success when asset_key is provided + with openai_resource.get_client_for_asset( + context=mock_context, asset_key=AssetKey("result") + ) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Say this is a test"}] + ) + + assert mock_client.called + assert mock_wrapper.call_count == 3 + mock_wrapper.assert_called_with( + api_endpoint_class=ANY, + context=mock_context, + output_name="result", + ) + + # Test failure when asset_key is not provided + with pytest.raises(DagsterInvariantViolationError): + with openai_resource.get_client(context=mock_context) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Say this is a test"}], + ) + return None, None + + result = materialize_to_memory( + [openai_multi_asset], + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + + assert result.success + + +@patch("dagster_openai.resources.OpenAIResource._wrap_with_usage_metadata") +@patch("dagster.AssetExecutionContext", autospec=AssetExecutionContext) +@patch("dagster_openai.resources.Client") +def test_openai_resource_with_partitioned_asset(mock_client, mock_context, mock_wrapper): + openai_partitions_def = StaticPartitionsDefinition([str(j) for j in range(5)]) + + openai_partitioned_assets = [] + + for i in range(5): + + @asset( + name=f"openai_partitioned_asset_{i}", + group_name="openai_partitioned_assets", + partitions_def=openai_partitions_def, + ) + def openai_partitioned_asset(openai_resource: OpenAIResource): + assert openai_resource + + mock_context.output_for_asset_key.return_value = "test" + + with openai_resource.get_client(context=mock_context) as client: + client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Say this is a test"}], + ) + + assert mock_client.called + assert mock_wrapper.called + mock_wrapper.assert_called_with( + api_endpoint_class=ANY, + context=mock_context, + output_name="test", + ) + + openai_partitioned_assets.append(openai_partitioned_asset) + + defs = Definitions( + assets=openai_partitioned_assets, + jobs=[ + define_asset_job( + name="openai_partitioned_asset_job", + selection=AssetSelection.groups("openai_partitioned_assets"), + partitions_def=openai_partitions_def, + ) + ], + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + + for partition_key in openai_partitions_def.get_partition_keys(): + result = defs.get_job_def("openai_partitioned_asset_job").execute_in_process( + partition_key=partition_key + ) + assert result.success + + expected_wrapper_call_counts = ( + 3 * len(openai_partitioned_assets) * len(openai_partitions_def.get_partition_keys()) + ) + assert mock_wrapper.call_count == expected_wrapper_call_counts + + +@patch("dagster.OpExecutionContext", autospec=OpExecutionContext) +@patch("dagster_openai.resources.Client") +def test_openai_wrapper_with_op(mock_client, mock_context): + @op + def openai_op(openai_resource: OpenAIResource): + assert openai_resource + + with openai_resource.get_client(context=mock_context) as client: + with pytest.raises(DagsterInvariantViolationError): + client.fine_tuning.jobs.create = with_usage_metadata( + context=mock_context, + output_name="some_output_name", + func=client.fine_tuning.jobs.create, + ) + + result = wrap_op_in_graph_and_execute( + openai_op, + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + assert result.success + + +@patch("dagster_openai.resources.OpenAIResource._wrap_with_usage_metadata") +@patch("dagster.AssetExecutionContext", autospec=AssetExecutionContext) +@patch("dagster_openai.resources.Client") +def test_openai_wrapper_with_asset(mock_client, mock_context, mock_wrapper): + @asset + def openai_asset(openai_resource: OpenAIResource): + assert openai_resource + + mock_completion = MagicMock() + mock_usage = MagicMock() + mock_usage.prompt_tokens = 1 + mock_usage.total_tokens = 1 + mock_usage.completion_tokens = 1 + mock_completion.usage = mock_usage + mock_client.return_value.fine_tuning.jobs.create.return_value = mock_completion + + with openai_resource.get_client(context=mock_context) as client: + client.fine_tuning.jobs.create = with_usage_metadata( + context=mock_context, + output_name="openai_asset", + func=client.fine_tuning.jobs.create, + ) + client.fine_tuning.jobs.create( + model="gpt-3.5-turbo", training_file="some_training_file" + ) + + mock_context.add_output_metadata.assert_called_with( + metadata={ + "openai.calls": 1, + "openai.total_tokens": 1, + "openai.prompt_tokens": 1, + "openai.completion_tokens": 1, + }, + output_name="openai_asset", + ) + + result = materialize_to_memory( + [openai_asset], + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + + assert result.success + + +@patch("dagster_openai.resources.OpenAIResource._wrap_with_usage_metadata") +@patch("dagster.AssetExecutionContext", autospec=AssetExecutionContext) +@patch("dagster_openai.resources.Client") +def test_openai_wrapper_with_graph_backed_asset(mock_client, mock_context, mock_wrapper): + @op + def model_version_op(): + return "gpt-3.5-turbo" + + @op + def training_file_op(): + return "some_training_file" + + @op + def openai_op(openai_resource: OpenAIResource, model_version, training_file): + assert openai_resource + + mock_completion = MagicMock() + mock_usage = MagicMock() + mock_usage.prompt_tokens = 1 + mock_usage.total_tokens = 1 + mock_usage.completion_tokens = 1 + mock_completion.usage = mock_usage + mock_client.return_value.fine_tuning.jobs.create.return_value = mock_completion + + with openai_resource.get_client(context=mock_context) as client: + client.fine_tuning.jobs.create = with_usage_metadata( + context=mock_context, + output_name="openai_asset", + func=client.fine_tuning.jobs.create, + ) + client.fine_tuning.jobs.create(model=model_version, training_file=training_file) + + mock_context.add_output_metadata.assert_called_with( + metadata={ + "openai.calls": 1, + "openai.total_tokens": 1, + "openai.prompt_tokens": 1, + "openai.completion_tokens": 1, + }, + output_name="openai_asset", + ) + + @graph_asset + def openai_asset(): + return openai_op(model_version_op(), training_file_op()) + + result = materialize_to_memory( + [openai_asset], + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + + assert result.success + + +@patch("dagster_openai.resources.OpenAIResource._wrap_with_usage_metadata") +@patch("dagster.AssetExecutionContext", autospec=AssetExecutionContext) +@patch("dagster_openai.resources.Client") +def test_openai_wrapper_with_multi_asset(mock_client, mock_context, mock_wrapper): + @multi_asset( + specs=[AssetSpec("status"), AssetSpec("result")], + ) + def openai_multi_asset(openai_resource: OpenAIResource): + assert openai_resource + + mock_completion = MagicMock() + mock_usage = MagicMock() + mock_usage.prompt_tokens = 1 + mock_usage.total_tokens = 1 + mock_usage.completion_tokens = 1 + mock_completion.usage = mock_usage + mock_client.return_value.fine_tuning.jobs.create.return_value = mock_completion + + with openai_resource.get_client_for_asset( + context=mock_context, asset_key=AssetKey("result") + ) as client: + client.fine_tuning.jobs.create = with_usage_metadata( + context=mock_context, + output_name="result", + func=client.fine_tuning.jobs.create, + ) + client.fine_tuning.jobs.create( + model="gpt-3.5-turbo", training_file="some_training_file" + ) + + mock_context.add_output_metadata.assert_called_with( + metadata={ + "openai.calls": 1, + "openai.total_tokens": 1, + "openai.prompt_tokens": 1, + "openai.completion_tokens": 1, + }, + output_name="result", + ) + return None, None + + result = materialize_to_memory( + [openai_multi_asset], + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + + assert result.success + + +@patch("dagster_openai.resources.OpenAIResource._wrap_with_usage_metadata") +@patch("dagster_openai.resources.Client") +def test_openai_wrapper_with_partitioned_asset(mock_client, mock_wrapper): + openai_partitions_def = StaticPartitionsDefinition([str(j) for j in range(5)]) + + openai_partitioned_assets = [] + + for i in range(5): + + @asset( + name=f"openai_partitioned_asset_{i}", + group_name="openai_partitioned_assets", + partitions_def=openai_partitions_def, + ) + def openai_partitioned_asset(openai_resource: OpenAIResource): + assert openai_resource + + mock_context = MagicMock() + mock_context.__class__ = AssetExecutionContext + + mock_completion = MagicMock() + mock_usage = MagicMock() + mock_usage.prompt_tokens = 1 + mock_usage.total_tokens = 1 + mock_usage.completion_tokens = 1 + mock_completion.usage = mock_usage + mock_client.return_value.fine_tuning.jobs.create.return_value = mock_completion + + with openai_resource.get_client(context=mock_context) as client: + client.fine_tuning.jobs.create = with_usage_metadata( + context=mock_context, + output_name=None, + func=client.fine_tuning.jobs.create, + ) + client.fine_tuning.jobs.create( + model="gpt-3.5-turbo", training_file="some_training_file" + ) + mock_context.add_output_metadata.assert_called_with( + { + "openai.calls": 1, + "openai.total_tokens": 1, + "openai.prompt_tokens": 1, + "openai.completion_tokens": 1, + }, + None, + ) + + openai_partitioned_assets.append(openai_partitioned_asset) + + defs = Definitions( + assets=openai_partitioned_assets, + jobs=[ + define_asset_job( + name="openai_partitioned_asset_job", + selection=AssetSelection.groups("openai_partitioned_assets"), + partitions_def=openai_partitions_def, + ) + ], + resources={ + "openai_resource": OpenAIResource(api_key="xoxp-1234123412341234-12341234-1234") + }, + ) + + for partition_key in openai_partitions_def.get_partition_keys(): + result = defs.get_job_def("openai_partitioned_asset_job").execute_in_process( + partition_key=partition_key + ) + assert result.success diff --git a/python_modules/libraries/dagster-openai/setup.cfg b/python_modules/libraries/dagster-openai/setup.cfg new file mode 100644 index 0000000000000..aa3e57c82a465 --- /dev/null +++ b/python_modules/libraries/dagster-openai/setup.cfg @@ -0,0 +1,9 @@ +[metadata] +license_files = LICENSE + +[check-manifest] +ignore = + .coveragerc + tox.ini + pytest.ini + dagster_openai_tests/** diff --git a/python_modules/libraries/dagster-openai/setup.py b/python_modules/libraries/dagster-openai/setup.py new file mode 100644 index 0000000000000..95a9f28fbb299 --- /dev/null +++ b/python_modules/libraries/dagster-openai/setup.py @@ -0,0 +1,40 @@ +from pathlib import Path + +from setuptools import find_packages, setup + + +def get_version(): + version = {} + with open(Path(__file__).parent / "dagster_openai/version.py", encoding="utf8") as fp: + exec(fp.read(), version) + + return version["__version__"] + + +# TODO: replace by `ver = get_version()` when we add back a version.py file and publish library to pypi. +ver = "1!0+dev" +# dont pin dev installs to avoid pip dep resolver issues +pin = "" if ver == "1!0+dev" else f"=={ver}" +setup( + name="dagster-openai", + version=ver, + author="Dagster Labs", + author_email="hello@dagsterlabs.com", + license="Apache-2.0", + description="A Open AI client resource for interacting with Open AI API.", + url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-openai", + classifiers=[ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + ], + packages=find_packages(exclude=["dagster_openai_tests*"]), + install_requires=[ + f"dagster{pin}", + "openai", + ], + zip_safe=False, +) diff --git a/python_modules/libraries/dagster-openai/tox.ini b/python_modules/libraries/dagster-openai/tox.ini new file mode 100644 index 0000000000000..6bfb5747cbacd --- /dev/null +++ b/python_modules/libraries/dagster-openai/tox.ini @@ -0,0 +1,15 @@ +[tox] +skipsdist = true + +[testenv] +download = True +passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* +deps = + -e ../../dagster[test] + -e ../../dagster-pipes + -e . +allowlist_externals = + /bin/bash +commands = + !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster' + pytest -c ../../../pyproject.toml -vv {posargs}