From a247a1a955998bafab6b72a90ef5198ff8448347 Mon Sep 17 00:00:00 2001 From: Sean Mackesey Date: Wed, 31 Jan 2024 13:09:53 -0500 Subject: [PATCH] fix type errors [INTERNAL_BRANCH=sean/pyright-pins] --- docs/content/integrations/embedded-elt.mdx | 2 +- .../concepts/resources/pythonic_resources.py | 10 ++-- .../pythonic_config/pythonic_config.py | 6 +- .../reference/pyspark_with_spark_session.py | 2 +- .../embedded_elt/postgres_snowflake.py | 2 +- .../integrations/embedded_elt/s3_snowflake.py | 2 +- .../intro_tutorial_tests/test_type_guide.py | 4 +- .../assets/core/id_range_for_time.py | 4 +- pyright/alt-1/requirements-pinned.txt | 28 +++++----- pyright/master/requirements-pinned.txt | 41 +++++++------- .../dagster_graphql/implementation/events.py | 4 +- .../graphql/test_execute_pipeline.py | 8 ++- .../dagster-pipes/dagster_pipes/__init__.py | 2 +- .../dagster_webserver/graphql.py | 2 +- .../dagster/dagster/_config/config_type.py | 2 +- .../dagster/dagster/_config/field_utils.py | 2 +- .../_config/pythonic_config/typing_utils.py | 26 +++++---- .../dagster/dagster/_config/validate.py | 2 +- .../decorators/asset_check_decorator.py | 6 +- .../_core/definitions/freshness_policy.py | 2 +- .../freshness_policy_sensor_definition.py | 11 ++-- .../_core/definitions/op_invocation.py | 8 +-- .../_core/definitions/schedule_definition.py | 2 +- .../_core/definitions/sensor_definition.py | 5 +- .../definitions/time_window_partitions.py | 4 +- .../_core/execution/context/compute.py | 2 +- .../dagster/_core/execution/host_mode.py | 2 +- .../host_representation/external_data.py | 2 +- .../dagster/_core/instance/__init__.py | 2 +- .../dagster/dagster/_core/log_manager.py | 2 +- .../dagster/dagster/_core/snap/node.py | 55 ++++++------------- .../dagster/_core/storage/db_io_manager.py | 14 +++-- .../_core/storage/event_log/sql_event_log.py | 2 +- .../dagster/dagster/_core/telemetry.py | 7 ++- .../dagster/dagster/_daemon/asset_daemon.py | 11 ++-- .../dagster/dagster/_daemon/sensor.py | 2 +- python_modules/dagster/dagster/_grpc/types.py | 19 ++++--- python_modules/dagster/dagster/_serdes/ipc.py | 12 ++-- .../dagster/dagster/_serdes/serdes.py | 27 ++++----- .../test_general_pythonic_resources.py | 4 +- .../test_type_signatures.py | 5 +- .../test_asset_check_decorator.py | 12 ++-- .../branching_io_manager_tests/utils.py | 50 ++++++++++++----- python_modules/dagster/setup.py | 4 +- .../dagster_airbyte/managed/reconciliation.py | 2 +- .../dagster_airbyte/managed/types.py | 3 +- .../dagster_airbyte/resources.py | 7 ++- .../test_dag_run_conf.py | 1 + .../databricks_pyspark_step_launcher.py | 2 +- .../dagstermill/examples/repository.py | 6 +- scripts/gen_airbyte_classes.py | 8 +-- 51 files changed, 242 insertions(+), 208 deletions(-) diff --git a/docs/content/integrations/embedded-elt.mdx b/docs/content/integrations/embedded-elt.mdx index 3ae6b34e8f6c6..ed0dca3ef4803 100644 --- a/docs/content/integrations/embedded-elt.mdx +++ b/docs/content/integrations/embedded-elt.mdx @@ -109,7 +109,7 @@ sling_job = build_assets_job( This is an example of how to setup a Sling sync between Postgres and Snowflake: ```python file=/integrations/embedded_elt/postgres_snowflake.py -# pyright: reportGeneralTypeIssues=none +# pyright: reportCallIssue=none # pyright: reportOptionalMemberAccess=none import os diff --git a/examples/docs_snippets/docs_snippets/concepts/resources/pythonic_resources.py b/examples/docs_snippets/docs_snippets/concepts/resources/pythonic_resources.py index fc293dfe7f548..cf55aabe0a2e8 100644 --- a/examples/docs_snippets/docs_snippets/concepts/resources/pythonic_resources.py +++ b/examples/docs_snippets/docs_snippets/concepts/resources/pythonic_resources.py @@ -289,7 +289,7 @@ class CredentialsResource(ConfigurableResource): password: str defs = Definitions( - assets=..., # type: ignore + assets=..., resources={ "credentials": CredentialsResource( username=EnvVar("MY_USERNAME"), @@ -375,7 +375,7 @@ def query(self, query: str): engine = create_engine(...) defs = Definitions( - assets=..., # type: ignore + assets=..., resources={"db": DBResource(engine=engine)}, ) @@ -475,7 +475,7 @@ def wrapped_io_manager(self) -> IOManagerDefinition: return old_file_io_manager defs = Definitions( - assets=..., # type: ignore + assets=..., resources={ "io_manager": MyIOManager(base_path="/tmp/"), }, @@ -553,7 +553,7 @@ def load_input(self, context: InputContext): return read_csv(self._get_path(context.asset_key)) defs = Definitions( - assets=..., # type: ignore + assets=..., resources={"io_manager": MyIOManager(root_path="/tmp/")}, ) @@ -669,7 +669,7 @@ class DBConnection: def query(self, body: str): ... - @contextmanager + @contextmanager # type: ignore def get_database_connection(username: str, password: str): ... diff --git a/examples/docs_snippets/docs_snippets/guides/dagster/pythonic_config/pythonic_config.py b/examples/docs_snippets/docs_snippets/guides/dagster/pythonic_config/pythonic_config.py index e1b81c3b70d1e..57de9e33b3ee1 100644 --- a/examples/docs_snippets/docs_snippets/guides/dagster/pythonic_config/pythonic_config.py +++ b/examples/docs_snippets/docs_snippets/guides/dagster/pythonic_config/pythonic_config.py @@ -167,12 +167,12 @@ def average_age(config: MyNestedConfig): "Alice": UserData( age=10, email="alice@gmail.com", - profile_picture_url=..., # type: ignore + profile_picture_url=..., ), "Bob": UserData( age=20, email="bob@gmail.com", - profile_picture_url=..., # type: ignore + profile_picture_url=..., ), } ) @@ -403,7 +403,7 @@ def required_config() -> None: class MyAssetConfig(Config): # ellipsis indicates that even though the type is Optional, # an input is required - person_first_name: Optional[str] = ... # type: ignore + person_first_name: Optional[str] = ... # ellipsis can also be used with pydantic.Field to attach metadata person_last_name: Optional[Callable] = Field( diff --git a/examples/docs_snippets/docs_snippets/integrations/duckdb/reference/pyspark_with_spark_session.py b/examples/docs_snippets/docs_snippets/integrations/duckdb/reference/pyspark_with_spark_session.py index 95bdef46e290c..4fefbdf195937 100644 --- a/examples/docs_snippets/docs_snippets/integrations/duckdb/reference/pyspark_with_spark_session.py +++ b/examples/docs_snippets/docs_snippets/integrations/duckdb/reference/pyspark_with_spark_session.py @@ -1,4 +1,4 @@ -# pyright: reportGeneralTypeIssues=none +# pyright: reportAttributeAccessIssue=none # start from dagster_duckdb_pyspark import DuckDBPySparkIOManager diff --git a/examples/docs_snippets/docs_snippets/integrations/embedded_elt/postgres_snowflake.py b/examples/docs_snippets/docs_snippets/integrations/embedded_elt/postgres_snowflake.py index 4f0e756b5ab07..cb667468d6c27 100644 --- a/examples/docs_snippets/docs_snippets/integrations/embedded_elt/postgres_snowflake.py +++ b/examples/docs_snippets/docs_snippets/integrations/embedded_elt/postgres_snowflake.py @@ -1,4 +1,4 @@ -# pyright: reportGeneralTypeIssues=none +# pyright: reportCallIssue=none # pyright: reportOptionalMemberAccess=none import os diff --git a/examples/docs_snippets/docs_snippets/integrations/embedded_elt/s3_snowflake.py b/examples/docs_snippets/docs_snippets/integrations/embedded_elt/s3_snowflake.py index 99e0dc7bde8a2..c8f8a66580f2c 100644 --- a/examples/docs_snippets/docs_snippets/integrations/embedded_elt/s3_snowflake.py +++ b/examples/docs_snippets/docs_snippets/integrations/embedded_elt/s3_snowflake.py @@ -1,4 +1,4 @@ -# pyright: reportGeneralTypeIssues=none +# pyright: reportCallIssue=none # pyright: reportOptionalMemberAccess=none import os diff --git a/examples/docs_snippets/docs_snippets_tests/intro_tutorial_tests/test_type_guide.py b/examples/docs_snippets/docs_snippets_tests/intro_tutorial_tests/test_type_guide.py index 9cc0a228875a9..49382a482e6d2 100755 --- a/examples/docs_snippets/docs_snippets_tests/intro_tutorial_tests/test_type_guide.py +++ b/examples/docs_snippets/docs_snippets_tests/intro_tutorial_tests/test_type_guide.py @@ -1,6 +1,6 @@ -# pyright: reportGeneralTypeIssues=none +# pyright: reportInvalidTypeForm=none -# Disable reportGeneralTypeIssues here due to use of Dagster types in annotations +# Disable reportInvalidTypeForm here due to use of Dagster types in annotations import typing diff --git a/examples/project_fully_featured/project_fully_featured/assets/core/id_range_for_time.py b/examples/project_fully_featured/project_fully_featured/assets/core/id_range_for_time.py index 64f9d4fc2c1ea..a2a3b26888d67 100644 --- a/examples/project_fully_featured/project_fully_featured/assets/core/id_range_for_time.py +++ b/examples/project_fully_featured/project_fully_featured/assets/core/id_range_for_time.py @@ -70,7 +70,9 @@ def _get_item_timestamp(item_id): min_item_id = hn_client.min_item_id() start_id = binary_search_nearest_left(_get_item_timestamp, min_item_id, max_item_id, start) - end_id = binary_search_nearest_right(_get_item_timestamp, min_item_id, max_item_id, end) + end_id = check.not_none( + binary_search_nearest_right(_get_item_timestamp, min_item_id, max_item_id, end) + ) start_timestamp = str(datetime.fromtimestamp(_get_item_timestamp(start_id), tz=timezone.utc)) end_timestamp = str(datetime.fromtimestamp(_get_item_timestamp(end_id), tz=timezone.utc)) diff --git a/pyright/alt-1/requirements-pinned.txt b/pyright/alt-1/requirements-pinned.txt index 08e47511551ac..01787dab94ca1 100644 --- a/pyright/alt-1/requirements-pinned.txt +++ b/pyright/alt-1/requirements-pinned.txt @@ -28,7 +28,7 @@ botocore==1.34.27 buildkite-test-collector==0.1.7 cachetools==5.3.2 caio==0.9.13 -certifi==2023.11.17 +certifi==2024.2.2 cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7 @@ -59,7 +59,7 @@ cycler==0.12.1 -e python_modules/libraries/dagster-snowflake-pyspark -e python_modules/libraries/dagster-spark -e python_modules/dagster-webserver -dbt-core==1.7.6 +dbt-core==1.7.7 dbt-duckdb==1.7.1 dbt-extractor==0.5.1 dbt-semantic-interfaces==0.4.3 @@ -82,7 +82,7 @@ fqdn==1.5.1 frozenlist==1.4.1 fsspec==2023.12.2 gcsfs==2023.12.2.post1 -google-api-core==2.16.0 +google-api-core==2.16.1 google-auth==2.27.0 google-auth-oauthlib==1.2.0 google-cloud-core==2.4.1 @@ -94,9 +94,9 @@ gql==3.5.0 graphene==3.3 graphql-core==3.2.3 graphql-relay==3.2.0 -grpcio==1.60.0 -grpcio-health-checking==1.60.0 -grpcio-tools==1.60.0 +grpcio==1.60.1 +grpcio-health-checking==1.60.1 +grpcio-tools==1.60.1 h11==0.14.0 httptools==0.6.1 humanfriendly==10.0 @@ -104,7 +104,7 @@ idna==3.6 importlib-metadata==6.11.0 iniconfig==2.0.0 ipykernel==6.29.0 -ipython==8.20.0 +ipython==8.21.0 isodate==0.6.1 isoduration==20.11.0 isort==5.13.2 @@ -144,7 +144,7 @@ mock==3.0.5 more-itertools==10.2.0 morefs==0.2.0 msgpack==1.0.7 -multidict==6.0.4 +multidict==6.0.5 multimethod==1.11 mypy==1.8.0 mypy-extensions==1.0.0 @@ -169,7 +169,7 @@ pandocfilters==1.5.1 parsedatetime==2.6 parso==0.8.3 pathspec==0.11.2 -pendulum==2.1.2 +pendulum==3.0.0 pexpect==4.9.0 pillow==10.2.0 platformdirs==3.11.0 @@ -184,7 +184,7 @@ ptyprocess==0.7.0 pure-eval==0.2.2 py==1.11.0 py4j==0.10.9.7 -pyarrow==10.0.1 +pyarrow==15.0.0 pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 @@ -207,10 +207,9 @@ pytest-xdist==3.3.1 python-dateutil==2.8.2 python-dotenv==1.0.1 python-json-logger==2.0.7 -python-slugify==8.0.2 +python-slugify==8.0.3 pytimeparse==1.1.8 -pytz==2023.4 -pytzdata==2020.1 +pytz==2024.1 PyYAML==6.0.1 pyzmq==25.1.2 referencing==0.33.0 @@ -232,7 +231,7 @@ Send2Trash==1.8.2 six==1.16.0 slack_sdk==3.26.2 sniffio==1.3.0 -snowflake-connector-python==3.4.1 +snowflake-connector-python==3.7.0 snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 soupsieve==2.5 @@ -246,6 +245,7 @@ tabulate==0.9.0 terminado==0.18.0 text-unidecode==1.3 threadpoolctl==3.2.0 +time-machine==2.13.0 tinycss2==1.2.1 toml==0.10.2 tomli==2.0.1 diff --git a/pyright/master/requirements-pinned.txt b/pyright/master/requirements-pinned.txt index 3cb12f5c531bb..a04f34a03634c 100644 --- a/pyright/master/requirements-pinned.txt +++ b/pyright/master/requirements-pinned.txt @@ -21,7 +21,7 @@ apache-airflow-providers-http==4.1.0 apache-airflow-providers-imap==3.5.0 apache-airflow-providers-sqlite==3.7.0 apeye==1.4.1 -apeye-core==1.1.4 +apeye-core==1.1.5 apispec==6.4.0 appdirs==1.4.4 appnope==0.1.3 @@ -57,10 +57,10 @@ bitmath==1.3.3.1 bleach==6.1.0 blinker==1.7.0 bokeh==3.3.4 -boto3==1.34.30 -botocore==1.34.30 +boto3==1.34.33 +botocore==1.34.33 buildkite-test-collector==0.1.7 -CacheControl==0.13.1 +CacheControl==0.14.0 cached-property==1.5.2 cachelib==0.9.0 cachetools==5.3.2 @@ -68,7 +68,7 @@ caio==0.9.13 callee==0.3.1 cattrs==23.2.3 celery==5.3.6 -certifi==2023.11.17 +certifi==2024.2.2 cffi==1.16.0 cfn-lint==0.85.0 chardet==5.2.0 @@ -219,8 +219,8 @@ fsspec==2023.12.2 future==0.18.3 gitdb==4.0.11 GitPython==3.1.41 -google-api-core==2.16.0 -google-api-python-client==2.115.0 +google-api-core==2.16.1 +google-api-python-client==2.116.0 google-auth==2.27.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.0 @@ -238,10 +238,10 @@ graphql-core==3.2.3 graphql-relay==3.2.0 graphviz==0.20.1 great-expectations==0.17.11 -grpcio==1.60.0 -grpcio-health-checking==1.60.0 -grpcio-status==1.60.0 -grpcio-tools==1.60.0 +grpcio==1.60.1 +grpcio-health-checking==1.60.1 +grpcio-status==1.60.1 +grpcio-tools==1.60.1 gunicorn==21.2.0 h11==0.14.0 hologram==0.0.16 @@ -259,7 +259,7 @@ importlib-resources==6.1.1 inflection==0.5.1 iniconfig==2.0.0 ipykernel==6.29.0 -ipython==8.20.0 +ipython==8.21.0 ipython-genutils==0.2.0 ipywidgets==8.1.1 iso8601==2.1.0 @@ -329,7 +329,7 @@ mpmath==1.3.0 msal==1.26.0 msal-extensions==1.1.0 msgpack==1.0.7 -multidict==6.0.4 +multidict==6.0.5 multimethod==1.11 mypy-extensions==1.0.0 mypy-protobuf==3.5.0 @@ -351,7 +351,7 @@ oauthlib==3.2.2 objgraph==3.6.0 onnx==1.15.0 onnxconverter-common==1.13.0 -onnxruntime==1.16.3 +onnxruntime==1.17.0 openapi-schema-validator==0.6.2 openapi-spec-validator==0.7.1 opentelemetry-api==1.22.0 @@ -406,7 +406,7 @@ pure-eval==0.2.2 py==1.11.0 py-partiql-parser==0.5.0 py4j==0.10.9.7 -pyarrow==10.0.1 +pyarrow==15.0.0 pyarrow-hotfix==0.6 pyasn1==0.5.1 pyasn1-modules==0.3.0 @@ -438,7 +438,7 @@ python-jose==3.3.0 python-json-logger==2.0.7 python-multipart==0.0.6 python-nvd3==0.15.0 -python-slugify==8.0.2 +python-slugify==8.0.3 python-utils==3.8.2 pytimeparse==1.1.8 pytz==2023.4 @@ -468,7 +468,7 @@ scikit-learn==1.4.0 scipy==1.12.0 scrapbook==0.5.0 seaborn==0.13.2 -selenium==4.17.1 +selenium==4.16.0 Send2Trash==1.8.2 sending==0.3.0 sentry-sdk==1.40.0 @@ -481,7 +481,7 @@ sling==1.0.72 smmap==5.0.1 sniffio==1.3.0 snowballstemmer==2.2.0 -snowflake-connector-python==3.4.1 +snowflake-connector-python==3.7.0 snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 soupsieve==2.5 @@ -549,14 +549,13 @@ types-pyOpenSSL==24.0.0.20240130 types-python-dateutil==2.8.19.20240106 types-pytz==2023.4.0.20240130 types-PyYAML==6.0.12.12 -types-requests==2.31.0.6 +types-requests==2.31.0.20240125 types-simplejson==3.19.0.2 types-six==1.16.21.20240106 types-SQLAlchemy==1.4.53.34 types-tabulate==0.9.0.20240106 types-toml==0.10.8.7 types-tzlocal==5.1.0.1 -types-urllib3==1.26.25.14 typing-inspect==0.9.0 typing_extensions==4.5.0 tzdata==2023.4 @@ -566,7 +565,7 @@ unicodecsv==0.14.1 universal-pathlib==0.1.4 uri-template==1.3.0 uritemplate==4.1.1 -urllib3==1.26.18 +urllib3==2.0.7 uvicorn==0.27.0.post1 uvloop==0.19.0 vine==5.1.0 diff --git a/python_modules/dagster-graphql/dagster_graphql/implementation/events.py b/python_modules/dagster-graphql/dagster_graphql/implementation/events.py index 8c1d74b69465f..6efdb6132312e 100644 --- a/python_modules/dagster-graphql/dagster_graphql/implementation/events.py +++ b/python_modules/dagster-graphql/dagster_graphql/implementation/events.py @@ -335,9 +335,7 @@ def from_dagster_event_record(event_record: EventLogEntry, pipeline_name: str) - return GrapheneHandledOutputEvent( output_name=data.output_name, manager_key=data.manager_key, - metadataEntries=_to_metadata_entries( - dagster_event.event_specific_data.metadata # type: ignore - ), + metadataEntries=_to_metadata_entries(dagster_event.event_specific_data.metadata), **basic_params, ) elif dagster_event.event_type == DagsterEventType.LOADED_INPUT: diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_execute_pipeline.py b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_execute_pipeline.py index f3f4ed980d5c2..658ff0a84a7ee 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_execute_pipeline.py +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_execute_pipeline.py @@ -1,6 +1,7 @@ import json import time import uuid +from typing import Any, Optional from dagster._core.storage.dagster_run import DagsterRunStatus, RunsFilter from dagster._core.test_utils import wait_for_runs_to_finish @@ -17,6 +18,7 @@ execute_dagster_graphql_subscription, infer_job_selector, ) +from typing_extensions import Dict from .graphql_context_test_suite import ( ExecutingGraphQLContextTestMatrix, @@ -581,9 +583,11 @@ def test_basic_filesystem_sync_execution(self, graphql_context: WorkspaceRequest assert has_event_of_type(logs, "RunSuccessEvent") assert not has_event_of_type(logs, "RunFailureEvent") - assert first_event_of_type(logs, "RunStartEvent")["level"] == "DEBUG" + run_start_event = first_event_of_type(logs, "RunStartEvent") + assert run_start_event and run_start_event["level"] == "DEBUG" sum_op_output = get_step_output_event(logs, "sum_op") + assert sum_op_output assert sum_op_output["stepKey"] == "sum_op" assert sum_op_output["outputName"] == "result" @@ -775,7 +779,7 @@ def _get_step_run_log_entry(pipeline_run_logs, step_key, typename): return message_data -def first_event_of_type(logs, message_type): +def first_event_of_type(logs, message_type) -> Optional[Dict[str, Any]]: for log in logs: if log["__typename"] == message_type: return log diff --git a/python_modules/dagster-pipes/dagster_pipes/__init__.py b/python_modules/dagster-pipes/dagster_pipes/__init__.py index 935d237a9ac9b..6e5856f50e9e4 100644 --- a/python_modules/dagster-pipes/dagster_pipes/__init__.py +++ b/python_modules/dagster-pipes/dagster_pipes/__init__.py @@ -501,7 +501,7 @@ def get_opened_extras(self) -> PipesExtras: return {} -class PipesMessageWriterChannel(ABC, Generic[T_MessageChannel]): +class PipesMessageWriterChannel(ABC): """Object that writes messages back to the Dagster orchestration process.""" @abstractmethod diff --git a/python_modules/dagster-webserver/dagster_webserver/graphql.py b/python_modules/dagster-webserver/dagster_webserver/graphql.py index f95af22296564..2bf0f40562b95 100644 --- a/python_modules/dagster-webserver/dagster_webserver/graphql.py +++ b/python_modules/dagster-webserver/dagster_webserver/graphql.py @@ -162,7 +162,7 @@ async def graphql_http_endpoint(self, request: Request): with ErrorCapture.watch(captured_errors.append): result = await self.execute_graphql_request(request, query, variables, operation_name) - response_data = {"data": result.data} + response_data: Dict[str, Any] = {"data": result.data} if result.errors: response_data["errors"] = self.handle_graphql_errors(result.errors) diff --git a/python_modules/dagster/dagster/_config/config_type.py b/python_modules/dagster/dagster/_config/config_type.py index 49c427d8849ff..25b5c78c2f437 100644 --- a/python_modules/dagster/dagster/_config/config_type.py +++ b/python_modules/dagster/dagster/_config/config_type.py @@ -466,7 +466,7 @@ def type_iterator(self) -> Iterator["ConfigType"]: ConfigIntInstance: Int = Int() ConfigStringInstance: String = String() -_CONFIG_MAP: Dict[check.TypeOrTupleOfTypes, ConfigType] = { +_CONFIG_MAP: Dict[typing.Any, ConfigType] = { BuiltinEnum.ANY: ConfigAnyInstance, BuiltinEnum.BOOL: ConfigBoolInstance, BuiltinEnum.FLOAT: ConfigFloatInstance, diff --git a/python_modules/dagster/dagster/_config/field_utils.py b/python_modules/dagster/dagster/_config/field_utils.py index cab376f934275..527e7bd776f64 100644 --- a/python_modules/dagster/dagster/_config/field_utils.py +++ b/python_modules/dagster/dagster/_config/field_utils.py @@ -411,7 +411,7 @@ def expand_map(original_root: object, the_dict: Mapping[object, object], stack: key = next(iter(the_dict.keys())) key_type = _convert_potential_type(original_root, key, stack) - if not key_type or not key_type.kind == ConfigTypeKind.SCALAR: + if not key_type or not key_type.kind == ConfigTypeKind.SCALAR: # type: ignore raise DagsterInvalidConfigDefinitionError( original_root, the_dict, diff --git a/python_modules/dagster/dagster/_config/pythonic_config/typing_utils.py b/python_modules/dagster/dagster/_config/pythonic_config/typing_utils.py index 24d09d5b1e3e4..ba2790b370a72 100644 --- a/python_modules/dagster/dagster/_config/pythonic_config/typing_utils.py +++ b/python_modules/dagster/dagster/_config/pythonic_config/typing_utils.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Any, Generic, Optional, Type, TypeVar, Union, cast from pydantic import Field -from typing_extensions import Annotated, dataclass_transform, get_origin +from typing_extensions import Annotated, Self, dataclass_transform, get_origin from dagster._core.errors import DagsterInvalidDagsterTypeInPythonicConfigDefinitionError @@ -138,9 +138,6 @@ def __new__(cls, name, bases, namespaces, **kwargs) -> Any: return super().__new__(cls, name, bases, namespaces, **kwargs) -T_Self = TypeVar("T_Self", bound="TypecheckAllowPartialResourceInitParams") - - class TypecheckAllowPartialResourceInitParams: """Implementation of the Python descriptor protocol (https://docs.python.org/3/howto/descriptor.html) to adjust the types of resource inputs and outputs, e.g. resource dependencies can be passed in @@ -176,12 +173,21 @@ class FooResource(Resource): def __set_name__(self, _owner, name): self._assigned_name = name - def __get__(self: "T_Self", obj: Any, __owner: Any) -> "T_Self": + def __get__(self: Self, obj: Any, __owner: Any) -> Self: # no-op implementation (only used to affect type signature) - return cast(T_Self, getattr(obj, self._assigned_name)) - - def __set__( - self: "T_Self", obj: Optional[object], value: Union["T_Self", "PartialResource[T_Self]"] - ) -> None: + return cast(Self, getattr(obj, self._assigned_name)) + + # The annotation her has been temporarily changed from: + # value: Union[Self, "PartialResource[Self]"] + # to: + # value: Union[Any, "PartialResource[Any]"] + # This is because of a bug in mypy that is incorrectly interpreting the + # signature and can cause a false positive type error for users. This only + # started being detected in our test_type_signatures.py tests on 2024-02-02 + # when some annotations elsewhere were added, likely causing mypy to + # analyze code it was previously skipping. The annotation should be + # reverted when the bug is fixed or another solution that surface as type + # errors for mypy users is found. + def __set__(self, obj: Optional[object], value: Union[Any, "PartialResource[Any]"]) -> None: # no-op implementation (only used to affect type signature) setattr(obj, self._assigned_name, value) diff --git a/python_modules/dagster/dagster/_config/validate.py b/python_modules/dagster/dagster/_config/validate.py index 4a75b40fd3feb..efcabad3b9df2 100644 --- a/python_modules/dagster/dagster/_config/validate.py +++ b/python_modules/dagster/dagster/_config/validate.py @@ -434,7 +434,7 @@ def validate_enum_config( def process_config( config_type: object, config_dict: Mapping[str, object] -) -> EvaluateValueResult[Mapping[str, object]]: +) -> EvaluateValueResult[Mapping[str, Any]]: config_type = resolve_to_config_type(config_type) config_type = check.inst(cast(ConfigType, config_type), ConfigType) validate_evr = validate_config(config_type, config_dict) diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/asset_check_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/asset_check_decorator.py index f9decc22583f6..4cf4f31fcb95a 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/asset_check_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/asset_check_decorator.py @@ -1,5 +1,7 @@ from typing import AbstractSet, Any, Callable, Iterable, Mapping, Optional, Set, Tuple, Union +from typing_extensions import TypeAlias + from dagster import _check as check from dagster._annotations import experimental from dagster._config import UserConfigSchema @@ -28,8 +30,8 @@ ) from .op_decorator import _Op -AssetCheckFunctionReturn = AssetCheckResult -AssetCheckFunction = Callable[..., AssetCheckFunctionReturn] +AssetCheckFunctionReturn: TypeAlias = AssetCheckResult +AssetCheckFunction: TypeAlias = Callable[..., AssetCheckFunctionReturn] def _build_asset_check_input( diff --git a/python_modules/dagster/dagster/_core/definitions/freshness_policy.py b/python_modules/dagster/dagster/_core/definitions/freshness_policy.py index ca7c2a40f5627..0ac565f480da9 100644 --- a/python_modules/dagster/dagster/_core/definitions/freshness_policy.py +++ b/python_modules/dagster/dagster/_core/definitions/freshness_policy.py @@ -121,7 +121,7 @@ def __new__( ) try: # Verify that the timezone can be loaded - pendulum_create_timezone(cron_schedule_timezone) # type: ignore + pendulum_create_timezone(cron_schedule_timezone) except Exception as e: raise DagsterInvalidDefinitionError( "Invalid cron schedule timezone '{cron_schedule_timezone}'. " diff --git a/python_modules/dagster/dagster/_core/definitions/freshness_policy_sensor_definition.py b/python_modules/dagster/dagster/_core/definitions/freshness_policy_sensor_definition.py index 202812592ed03..83dc6bcf4ad60 100644 --- a/python_modules/dagster/dagster/_core/definitions/freshness_policy_sensor_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/freshness_policy_sensor_definition.py @@ -27,6 +27,7 @@ from .sensor_definition import ( DefaultSensorStatus, + RawSensorEvaluationFunctionReturn, SensorDefinition, SensorEvaluationContext, SensorType, @@ -200,7 +201,7 @@ def __init__( self, name: str, asset_selection: AssetSelection, - freshness_policy_sensor_fn: Callable[..., None], + freshness_policy_sensor_fn: Callable[..., RawSensorEvaluationFunctionReturn], minimum_interval_seconds: Optional[int] = None, description: Optional[str] = None, default_status: DefaultSensorStatus = DefaultSensorStatus.STOPPED, @@ -312,7 +313,7 @@ def _wrapped_fn(context: SensorEvaluationContext): required_resource_keys=combined_required_resource_keys, ) - def __call__(self, *args, **kwargs) -> None: + def __call__(self, *args, **kwargs) -> RawSensorEvaluationFunctionReturn: context_param_name = get_context_param_name(self._freshness_policy_sensor_fn) sensor_context = get_sensor_context_from_args_or_kwargs( @@ -347,7 +348,7 @@ def freshness_policy_sensor( description: Optional[str] = None, default_status: DefaultSensorStatus = DefaultSensorStatus.STOPPED, ) -> Callable[ - [Callable[..., None]], + [Callable[..., RawSensorEvaluationFunctionReturn]], FreshnessPolicySensorDefinition, ]: """Define a sensor that reacts to the status of a given set of asset freshness policies, where the @@ -370,7 +371,9 @@ def freshness_policy_sensor( status can be overridden from the Dagster UI or via the GraphQL API. """ - def inner(fn: Callable[..., None]) -> FreshnessPolicySensorDefinition: + def inner( + fn: Callable[..., RawSensorEvaluationFunctionReturn], + ) -> FreshnessPolicySensorDefinition: check.callable_param(fn, "fn") sensor_name = name or fn.__name__ diff --git a/python_modules/dagster/dagster/_core/definitions/op_invocation.py b/python_modules/dagster/dagster/_core/definitions/op_invocation.py index d77dd6eb81d3a..d5b8d4d8dbe07 100644 --- a/python_modules/dagster/dagster/_core/definitions/op_invocation.py +++ b/python_modules/dagster/dagster/_core/definitions/op_invocation.py @@ -222,11 +222,11 @@ def direct_invocation_result( # try-except handles "vanilla" asset and op invocation (generators and async handled in # _type_check_output_wrapper) - input_dict = _resolve_inputs(op_def, input_args, input_kwargs, bound_context) + input_dict = _resolve_inputs(op_def, input_args, input_kwargs, bound_context) # type: ignore (pyright bug) result = invoke_compute_fn( fn=compute_fn.decorated_fn, - context=bound_context, + context=bound_context, # type: ignore (pyright bug) kwargs=input_dict, context_arg_provided=compute_fn.has_context_arg(), config_arg_cls=( @@ -234,9 +234,9 @@ def direct_invocation_result( ), resource_args=resource_arg_mapping, ) - return _type_check_output_wrapper(op_def, result, bound_context) + return _type_check_output_wrapper(op_def, result, bound_context) # type: ignore (pyright bug) except Exception: - bound_context.unbind() + bound_context.unbind() # type: ignore (pyright bug) raise diff --git a/python_modules/dagster/dagster/_core/definitions/schedule_definition.py b/python_modules/dagster/dagster/_core/definitions/schedule_definition.py index 2cd6b39bcad8c..766594a1fe0d3 100644 --- a/python_modules/dagster/dagster/_core/definitions/schedule_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/schedule_definition.py @@ -60,7 +60,7 @@ RunRequestIterator: TypeAlias = Iterator[Union[RunRequest, SkipReason]] ScheduleEvaluationFunctionReturn: TypeAlias = Union[ - RunRequest, SkipReason, RunConfig, RunRequestIterator, Sequence[RunRequest] + RunRequest, SkipReason, RunConfig, RunRequestIterator, Sequence[RunRequest], None ] RawScheduleEvaluationFunction: TypeAlias = Callable[..., ScheduleEvaluationFunctionReturn] diff --git a/python_modules/dagster/dagster/_core/definitions/sensor_definition.py b/python_modules/dagster/dagster/_core/definitions/sensor_definition.py index f3445da6262fc..6a081776c0da7 100644 --- a/python_modules/dagster/dagster/_core/definitions/sensor_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/sensor_definition.py @@ -440,10 +440,13 @@ def log_key(self) -> Optional[Sequence[str]]: RunRequest, DagsterRunReaction, SensorResult, + None, ] RawSensorEvaluationFunction: TypeAlias = Callable[..., RawSensorEvaluationFunctionReturn] -SensorEvaluationFunction: TypeAlias = Callable[..., Sequence[Union[SkipReason, RunRequest]]] +SensorEvaluationFunction: TypeAlias = Callable[ + ..., Sequence[Union[None, SensorResult, SkipReason, RunRequest]] +] def get_context_param_name(fn: Callable) -> Optional[str]: diff --git a/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py b/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py index 4cae6baebdb36..bd649af90f3f2 100644 --- a/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py +++ b/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py @@ -1782,9 +1782,9 @@ def _add_partitions_to_time_windows( break else: if result_windows and window_start_timestamp == result_windows[0].start.timestamp(): - result_windows[0] = TimeWindow(window.start, included_window.end) # type: ignore + result_windows[0] = TimeWindow(window.start, included_window.end) elif result_windows and window.end == result_windows[0].start: - result_windows[0] = TimeWindow(window.start, included_window.end) # type: ignore + result_windows[0] = TimeWindow(window.start, included_window.end) else: result_windows.insert(0, window) diff --git a/python_modules/dagster/dagster/_core/execution/context/compute.py b/python_modules/dagster/dagster/_core/execution/context/compute.py index d1e921954c01e..636fd9432ce8b 100644 --- a/python_modules/dagster/dagster/_core/execution/context/compute.py +++ b/python_modules/dagster/dagster/_core/execution/context/compute.py @@ -1386,7 +1386,7 @@ def _copy_docs_from_op_execution_context(obj): ] -def _get_deprecation_kwargs(attr: str): +def _get_deprecation_kwargs(attr: str) -> Mapping[str, Any]: deprecation_kwargs = {"breaking_version": "1.8.0"} deprecation_kwargs["subject"] = f"AssetExecutionContext.{attr}" diff --git a/python_modules/dagster/dagster/_core/execution/host_mode.py b/python_modules/dagster/dagster/_core/execution/host_mode.py index 99284748c9248..f6b76d5af0fb0 100644 --- a/python_modules/dagster/dagster/_core/execution/host_mode.py +++ b/python_modules/dagster/dagster/_core/execution/host_mode.py @@ -62,7 +62,7 @@ def _get_host_mode_executor( init_context = InitExecutorContext( job=recon_job, executor_def=executor_def, - executor_config=executor_config["config"], # type: ignore # (config typing) + executor_config=executor_config["config"], instance=instance, ) check_cross_process_constraints(init_context) diff --git a/python_modules/dagster/dagster/_core/host_representation/external_data.py b/python_modules/dagster/dagster/_core/host_representation/external_data.py index 18079c42d5a26..2a1c58267232f 100644 --- a/python_modules/dagster/dagster/_core/host_representation/external_data.py +++ b/python_modules/dagster/dagster/_core/host_representation/external_data.py @@ -1173,7 +1173,7 @@ class ExternalAssetNode( ("depended_by", Sequence[ExternalAssetDependedBy]), ("compute_kind", Optional[str]), ("op_name", Optional[str]), - ("op_names", Optional[Sequence[str]]), + ("op_names", Sequence[str]), ("code_version", Optional[str]), ("node_definition_name", Optional[str]), ("graph_name", Optional[str]), diff --git a/python_modules/dagster/dagster/_core/instance/__init__.py b/python_modules/dagster/dagster/_core/instance/__init__.py index d34cb3f45997b..e4e40d135c317 100644 --- a/python_modules/dagster/dagster/_core/instance/__init__.py +++ b/python_modules/dagster/dagster/_core/instance/__init__.py @@ -2685,7 +2685,7 @@ def scheduler_debug_info(self) -> "SchedulerDebugInfo": schedule_info: Mapping[str, Mapping[str, object]] = { schedule_state.instigator_name: { "status": schedule_state.status.value, - "cron_schedule": schedule_state.instigator_data.cron_schedule, + "cron_schedule": schedule_state.instigator_data.cron_schedule, # type: ignore "schedule_origin_id": schedule_state.instigator_origin_id, "repository_origin_id": schedule_state.repository_origin_id, } diff --git a/python_modules/dagster/dagster/_core/log_manager.py b/python_modules/dagster/dagster/_core/log_manager.py index 206adf4b466a4..c394d80060a49 100644 --- a/python_modules/dagster/dagster/_core/log_manager.py +++ b/python_modules/dagster/dagster/_core/log_manager.py @@ -31,7 +31,7 @@ def dagster_meta(self) -> "DagsterLoggingMetadata": # `extra` argument to `logging.Logger.log` (see `DagsterLogManager.log_dagster_event`), but # `logging.LogRecord` has no way of exposing to the type-checker the attributes that are dynamically # defined via `extra`. -class DagsterLogRecord(logging.LogRecord, IDagsterMeta): # type: ignore +class DagsterLogRecord(logging.LogRecord, IDagsterMeta): pass diff --git a/python_modules/dagster/dagster/_core/snap/node.py b/python_modules/dagster/dagster/_core/snap/node.py index 53fc531c09086..c03810b658436 100644 --- a/python_modules/dagster/dagster/_core/snap/node.py +++ b/python_modules/dagster/dagster/_core/snap/node.py @@ -181,29 +181,6 @@ def build_output_def_snap(output_def: OutputDefinition) -> OutputDefSnap: ) -# This and a set of shared props helps implement a de facto mixin for -# Inheritance is quite difficult and counterintuitive in namedtuple land, so went with this scheme -# instead. -def _check_node_def_header_args( - name: str, - input_def_snaps: Sequence[InputDefSnap], - output_def_snaps: Sequence[OutputDefSnap], - description: Optional[str], - tags: Mapping[str, str], - config_field_snap: Optional[ConfigFieldSnap], -): - return dict( - name=check.str_param(name, "name"), - input_def_snaps=check.sequence_param(input_def_snaps, "input_def_snaps", InputDefSnap), - output_def_snaps=check.sequence_param(output_def_snaps, "output_def_snaps", OutputDefSnap), - description=check.opt_str_param(description, "description"), - tags=check.mapping_param(tags, "tags"), - config_field_snap=check.opt_inst_param( - config_field_snap, "config_field_snap", ConfigFieldSnap - ), - ) - - @whitelist_for_serdes(storage_name="CompositeSolidDefSnap") class GraphDefSnap( NamedTuple( @@ -244,13 +221,15 @@ def __new__( output_mapping_snaps=check.sequence_param( output_mapping_snaps, "output_mapping_snaps", of_type=OutputMappingSnap ), - **_check_node_def_header_args( - name, - input_def_snaps, - output_def_snaps, - description, - tags, - config_field_snap, + name=check.str_param(name, "name"), + input_def_snaps=check.sequence_param(input_def_snaps, "input_def_snaps", InputDefSnap), + output_def_snaps=check.sequence_param( + output_def_snaps, "output_def_snaps", OutputDefSnap + ), + description=check.opt_str_param(description, "description"), + tags=check.mapping_param(tags, "tags"), + config_field_snap=check.opt_inst_param( + config_field_snap, "config_field_snap", ConfigFieldSnap ), ) @@ -291,13 +270,15 @@ def __new__( required_resource_keys=check.sequence_param( required_resource_keys, "required_resource_keys", str ), - **_check_node_def_header_args( - name, - input_def_snaps, - output_def_snaps, - description, - tags, - config_field_snap, + name=check.str_param(name, "name"), + input_def_snaps=check.sequence_param(input_def_snaps, "input_def_snaps", InputDefSnap), + output_def_snaps=check.sequence_param( + output_def_snaps, "output_def_snaps", OutputDefSnap + ), + description=check.opt_str_param(description, "description"), + tags=check.mapping_param(tags, "tags"), + config_field_snap=check.opt_inst_param( + config_field_snap, "config_field_snap", ConfigFieldSnap ), ) diff --git a/python_modules/dagster/dagster/_core/storage/db_io_manager.py b/python_modules/dagster/dagster/_core/storage/db_io_manager.py index ffd1f49c6cee2..175e8d5644a5b 100644 --- a/python_modules/dagster/dagster/_core/storage/db_io_manager.py +++ b/python_modules/dagster/dagster/_core/storage/db_io_manager.py @@ -4,6 +4,7 @@ Any, Dict, Generic, + Iterator, List, Mapping, NamedTuple, @@ -64,10 +65,10 @@ def supported_types(self) -> Sequence[Type[object]]: pass -class DbClient: +class DbClient(Generic[T]): @staticmethod @abstractmethod - def delete_table_slice(context: OutputContext, table_slice: TableSlice, connection) -> None: + def delete_table_slice(context: OutputContext, table_slice: TableSlice, connection: T) -> None: ... @staticmethod @@ -77,12 +78,17 @@ def get_select_statement(table_slice: TableSlice) -> str: @staticmethod @abstractmethod - def ensure_schema_exists(context: OutputContext, table_slice: TableSlice, connection) -> None: + def ensure_schema_exists( + context: OutputContext, table_slice: TableSlice, connection: T + ) -> None: ... @staticmethod + @abstractmethod @contextmanager - def connect(context: Union[OutputContext, InputContext], table_slice: TableSlice): + def connect( + context: Union[OutputContext, InputContext], table_slice: TableSlice + ) -> Iterator[T]: ... diff --git a/python_modules/dagster/dagster/_core/storage/event_log/sql_event_log.py b/python_modules/dagster/dagster/_core/storage/event_log/sql_event_log.py index 2af5954fcb4fc..7480a379e6a1b 100644 --- a/python_modules/dagster/dagster/_core/storage/event_log/sql_event_log.py +++ b/python_modules/dagster/dagster/_core/storage/event_log/sql_event_log.py @@ -1394,7 +1394,7 @@ def _fetch_asset_rows( should_query = bool(has_more) and bool(limit) and len(result) < cast(int, limit) is_partial_query = asset_keys is not None or bool(prefix) or bool(limit) or bool(cursor) - if not is_partial_query and self._can_mark_assets_as_migrated(rows): # type: ignore + if not is_partial_query and self._can_mark_assets_as_migrated(rows): self.enable_secondary_index(ASSET_KEY_INDEX_COLS) return result[:limit] if limit else result diff --git a/python_modules/dagster/dagster/_core/telemetry.py b/python_modules/dagster/dagster/_core/telemetry.py index e09f4ff9323f6..80b00ca377af7 100644 --- a/python_modules/dagster/dagster/_core/telemetry.py +++ b/python_modules/dagster/dagster/_core/telemetry.py @@ -28,6 +28,7 @@ NamedTuple, Optional, Sequence, + Tuple, TypeVar, Union, overload, @@ -396,7 +397,9 @@ def write_telemetry_log_line(log_line: object) -> None: logger.info(json.dumps(log_line)) -def _get_instance_telemetry_info(instance: DagsterInstance): +def _get_instance_telemetry_info( + instance: DagsterInstance, +) -> Tuple[bool, Optional[str], Optional[str]]: from dagster._core.storage.runs import SqlRunStorage check.inst_param(instance, "instance", DagsterInstance) @@ -741,7 +744,7 @@ def log_action( client_time=str(client_time), elapsed_time=str(elapsed_time), event_id=str(uuid.uuid4()), - instance_id=instance_id, + instance_id=check.not_none(instance_id), metadata=metadata, run_storage_id=run_storage_id, )._asdict() diff --git a/python_modules/dagster/dagster/_daemon/asset_daemon.py b/python_modules/dagster/dagster/_daemon/asset_daemon.py index b6390cca2cc67..06f0e3ad09f9e 100644 --- a/python_modules/dagster/dagster/_daemon/asset_daemon.py +++ b/python_modules/dagster/dagster/_daemon/asset_daemon.py @@ -515,11 +515,12 @@ def _run_iteration_impl( all_auto_materialize_states = {} for sensor in sensors: - selector_id = sensor.selector.get_id() if sensor else None - - auto_materialize_state = ( - all_auto_materialize_states.get(selector_id) if selector_id else None - ) + if sensor: + selector_id = sensor.selector.get_id() + auto_materialize_state = all_auto_materialize_states.get(selector_id) + else: + selector_id = None + auto_materialize_state = None if not sensor: # make sure we are only running every pre_sensor_interval_seconds diff --git a/python_modules/dagster/dagster/_daemon/sensor.py b/python_modules/dagster/dagster/_daemon/sensor.py index 45e731df67cae..807902da0d5d4 100644 --- a/python_modules/dagster/dagster/_daemon/sensor.py +++ b/python_modules/dagster/dagster/_daemon/sensor.py @@ -316,7 +316,7 @@ def execute_sensor_iteration( for sensor_state in instance.all_instigator_state(instigator_type=InstigatorType.SENSOR) if ( not sensor_state.instigator_data - or sensor_state.instigator_data.sensor_type != SensorType.AUTOMATION_POLICY + or sensor_state.instigator_data.sensor_type != SensorType.AUTOMATION_POLICY # type: ignore ) } diff --git a/python_modules/dagster/dagster/_grpc/types.py b/python_modules/dagster/dagster/_grpc/types.py index 6b6305d5a0010..1f260646762d5 100644 --- a/python_modules/dagster/dagster/_grpc/types.py +++ b/python_modules/dagster/dagster/_grpc/types.py @@ -130,7 +130,8 @@ def __new__( ) def get_command_args(self) -> Sequence[str]: - return _get_entry_point(self.job_origin) + [ + return [ + *_get_entry_point(self.job_origin), "api", "execute_run", serialize_value(self), @@ -180,7 +181,8 @@ def __new__( ) def get_command_args(self) -> Sequence[str]: - return _get_entry_point(self.job_origin) + [ + return [ + *_get_entry_point(self.job_origin), "api", "resume_run", serialize_value(self), @@ -281,15 +283,16 @@ def get_command_args(self, skip_serialized_namedtuple: bool = False) -> Sequence """Get the command args to run this step. If skip_serialized_namedtuple is True, then get_command_env should be used to pass the args to Click using an env var. """ - return ( - _get_entry_point(self.job_origin) - + ["api", "execute_step"] - + ( + return [ + *_get_entry_point(self.job_origin), + "api", + "execute_step", + *( ["--compressed-input-json", self._get_compressed_args()] if not skip_serialized_namedtuple else [] - ) - ) + ), + ] def get_command_env(self) -> Sequence[Mapping[str, str]]: """Get the env vars for overriding the Click args of this step. Used in conjuction with diff --git a/python_modules/dagster/dagster/_serdes/ipc.py b/python_modules/dagster/dagster/_serdes/ipc.py index 33c0f50335fee..c5b71914aa4f2 100644 --- a/python_modules/dagster/dagster/_serdes/ipc.py +++ b/python_modules/dagster/dagster/_serdes/ipc.py @@ -42,7 +42,7 @@ def ipc_write_unary_response(output_file: str, obj: NamedTuple) -> None: def read_unary_response( - output_file: str, timeout: int = 30, ipc_process: "Optional[Popen[bytes]]" = None + output_file: str, timeout: int = 30, ipc_process: "Optional[Popen[Any]]" = None ) -> Optional[NamedTuple]: messages = list(ipc_read_event_stream(output_file, timeout=timeout, ipc_process=ipc_process)) check.invariant(len(messages) == 1) @@ -139,7 +139,7 @@ def _process_line(file_pointer: TextIOWrapper, sleep_interval: float = 0.1) -> O sleep(sleep_interval) -def _poll_process(ipc_process: "Optional[Popen[bytes]]") -> None: +def _poll_process(ipc_process: "Optional[Popen[Any]]") -> None: if not ipc_process: return if ipc_process.poll() is not None: @@ -149,7 +149,7 @@ def _poll_process(ipc_process: "Optional[Popen[bytes]]") -> None: def ipc_read_event_stream( - file_path: str, timeout: int = 30, ipc_process: "Optional[Popen[bytes]]" = None + file_path: str, timeout: int = 30, ipc_process: "Optional[Popen[Any]]" = None ) -> Iterator[Optional[NamedTuple]]: # Wait for file to be ready sleep_interval = 0.1 @@ -191,7 +191,7 @@ def ipc_read_event_stream( # https://stefan.sofa-rockers.org/2013/08/15/handling-sub-process-hierarchies-python-linux-os-x/ -def open_ipc_subprocess(parts: Sequence[str], **kwargs: Any) -> "Popen[bytes]": +def open_ipc_subprocess(parts: Sequence[str], **kwargs: Any) -> "Popen[Any]": """Sets the correct flags to support graceful termination.""" check.list_param(parts, "parts", str) @@ -199,14 +199,14 @@ def open_ipc_subprocess(parts: Sequence[str], **kwargs: Any) -> "Popen[bytes]": if sys.platform == "win32": creationflags = subprocess.CREATE_NEW_PROCESS_GROUP - return subprocess.Popen( # type: ignore # (unclear whether this is actually guaranteed to return Popen[bytes]) + return subprocess.Popen( parts, creationflags=creationflags, **kwargs, ) -def interrupt_ipc_subprocess(proc: "Popen[bytes]") -> None: +def interrupt_ipc_subprocess(proc: "Popen[Any]") -> None: """Send CTRL_BREAK on Windows, SIGINT on other platforms.""" if sys.platform == "win32": proc.send_signal(signal.CTRL_BREAK_EVENT) diff --git a/python_modules/dagster/dagster/_serdes/serdes.py b/python_modules/dagster/dagster/_serdes/serdes.py index 9822d2a8f28f3..85a74c14c7a9a 100644 --- a/python_modules/dagster/dagster/_serdes/serdes.py +++ b/python_modules/dagster/dagster/_serdes/serdes.py @@ -20,6 +20,7 @@ from functools import partial from inspect import Parameter, signature from typing import ( + TYPE_CHECKING, AbstractSet, Any, Callable, @@ -31,7 +32,6 @@ Mapping, NamedTuple, Optional, - Protocol, Sequence, Set, Tuple, @@ -52,17 +52,16 @@ from .errors import DeserializationError, SerdesUsageError, SerializationError +if TYPE_CHECKING: + # There is no actual class backing Dataclasses, _typeshed provides this + # protocol. + from _typeshed import DataclassInstance + ################################################################################################### # Types ################################################################################################### -# dataclasses don't have a backing class, so use a Protocol to type them -class DataclassProtocol(Protocol): - __dict__: Mapping[str, Any] - __dataclass_fields__: Dict[str, dataclasses.Field] - - JsonSerializableValue: TypeAlias = Union[ Sequence["JsonSerializableValue"], Mapping[str, "JsonSerializableValue"], @@ -83,7 +82,7 @@ class DataclassProtocol(Protocol): None, NamedTuple, pydantic.BaseModel, - DataclassProtocol, + "DataclassInstance", Set["PackableValue"], FrozenSet["PackableValue"], Enum, @@ -99,7 +98,7 @@ class DataclassProtocol(Protocol): None, NamedTuple, pydantic.BaseModel, - DataclassProtocol, + "DataclassInstance", Set["PackableValue"], FrozenSet["PackableValue"], Enum, @@ -655,7 +654,7 @@ def constructor_param_names(self) -> Sequence[str]: return list(signature(self.klass.__new__).parameters.keys()) -T_Dataclass = TypeVar("T_Dataclass", bound=DataclassProtocol, default=DataclassProtocol) +T_Dataclass = TypeVar("T_Dataclass", bound="DataclassInstance", default="DataclassInstance") class DataclassSerializer(ObjectSerializer[T_Dataclass]): @@ -760,7 +759,7 @@ def pack_value( Set[PackableValue], FrozenSet[PackableValue], NamedTuple, - DataclassProtocol, + "DataclassInstance", pydantic.BaseModel, Enum, ], @@ -959,7 +958,7 @@ def __init__(self, message: str, value: Mapping[str, UnpackedValue]): self.value = value -def _unpack_object(val: dict, whitelist_map: WhitelistMap, context: UnpackContext): +def _unpack_object(val: dict, whitelist_map: WhitelistMap, context: UnpackContext) -> UnpackedValue: if "__class__" in val: klass_name = cast(str, val["__class__"]) if not whitelist_map.has_object_deserializer(klass_name): @@ -998,7 +997,9 @@ def _unpack_object(val: dict, whitelist_map: WhitelistMap, context: UnpackContex if "__mapping_items__" in val: return { - _unpack_value(k, whitelist_map, context): _unpack_value(v, whitelist_map, context) + cast(str, _unpack_value(k, whitelist_map, context)): _unpack_value( + v, whitelist_map, context + ) for k, v in val["__mapping_items__"] } diff --git a/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_general_pythonic_resources.py b/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_general_pythonic_resources.py index cc4db704b79eb..2239cc0b6f67a 100644 --- a/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_general_pythonic_resources.py +++ b/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_general_pythonic_resources.py @@ -1102,6 +1102,6 @@ class MergeResource(ConfigurableResource): # this should fail but does not https://github.com/dagster-io/dagster/issues/18017 MergeResource( - int_res=StrResource.configure_at_launch(), # type: ignore # type checker catches it though - str_res=IntResource.configure_at_launch(), # type: ignore + int_res=StrResource.configure_at_launch(), + str_res=IntResource.configure_at_launch(), ) diff --git a/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_type_signatures.py b/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_type_signatures.py index 126db0f7ded1c..445ba3478ed85 100644 --- a/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_type_signatures.py +++ b/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_type_signatures.py @@ -54,8 +54,7 @@ class OuterResource(ConfigurableResource): # Ensure constructor signature is correct (mypy doesn't yet support Pydantic model constructor type hints) assert pyright_out[0] == "(self: InnerResource, *, a_string: str) -> None" assert ( - pyright_out[1] - == "(self: OuterResource, *, inner: InnerResource | PartialResource[InnerResource]," + pyright_out[1] == "(self: OuterResource, *, inner: Any | PartialResource[Any]," " a_bool: bool) -> None" ) @@ -137,7 +136,7 @@ def test_type_signatures_alias(): from pydantic import Field class ResourceWithAlias(ConfigurableResource): - _schema: str = Field(alias="schema") + my_schema: str = Field(alias="schema") reveal_type(ResourceWithAlias.__init__) diff --git a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py index fde8d3b103c4c..0b22f1561985a 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py @@ -40,7 +40,7 @@ def execute_assets_and_checks( def test_asset_check_decorator(): @asset_check(asset="asset1", description="desc") def check1(): - ... + return AssetCheckResult(passed=True) assert check1.name == "check1" assert check1.description == "desc" @@ -50,7 +50,7 @@ def check1(): def test_asset_check_decorator_name(): @asset_check(asset="asset1", description="desc", name="check1") def _check(): - ... + return AssetCheckResult(passed=True) assert _check.name == "check1" @@ -62,7 +62,7 @@ def asset1(): @asset_check(asset=asset1) def my_check(): - ... + return AssetCheckResult(passed=True) assert my_check.asset_key == AssetKey(["prefix", "asset1"]) @@ -74,7 +74,7 @@ def asset1(): @asset_check(asset=asset1) def my_check(asset1): - ... + return AssetCheckResult(passed=True) assert my_check.asset_key == AssetKey(["prefix", "asset1"]) @@ -400,7 +400,7 @@ def test_definitions_same_name_different_asset(): def make_check_for_asset(asset_key: str): @asset_check(asset=asset_key) def check1(context: AssetExecutionContext): - ... + return AssetCheckResult(passed=True) return check1 @@ -411,7 +411,7 @@ def test_definitions_same_asset_different_name(): def make_check(check_name: str): @asset_check(asset="asset1", name=check_name) def _check(context: AssetExecutionContext): - ... + return AssetCheckResult(passed=True) return _check diff --git a/python_modules/dagster/dagster_tests/storage_tests/branching_io_manager_tests/utils.py b/python_modules/dagster/dagster_tests/storage_tests/branching_io_manager_tests/utils.py index dbfd6d85de2b2..cfb452194e7d9 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/branching_io_manager_tests/utils.py +++ b/python_modules/dagster/dagster_tests/storage_tests/branching_io_manager_tests/utils.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from typing import Any, Dict, List, Optional, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union from dagster import ( AssetKey, @@ -86,14 +86,22 @@ def __init__(self): def handle_output(self, context: OutputContext, obj: Any): keys = self._keys_from_context(context) - for key in keys: - self.values[key] = obj + if keys is None: + self.values[None] = obj + else: + for key in keys: + self.values[key] = obj def load_input(self, context: InputContext) -> Any: keys = self._keys_from_context(context) - return ( - {key[-1]: self.values[key] for key in keys} if len(keys) > 1 else self.values[keys[0]] - ) + if keys is None: + return self.values[None] + else: + return ( + {key[-1]: self.values[key] for key in keys} + if len(keys) > 1 + else self.values[keys[0]] + ) def has_value( self, asset_key: CoercibleToAssetKey, partition_key: Optional[str] = None @@ -103,9 +111,11 @@ def has_value( def get_value(self, asset_key: CoercibleToAssetKey, partition_key: Optional[str] = None) -> Any: return self.values.get(self._get_key(AssetKey.from_coercible(asset_key), partition_key)) - def _keys_from_context(self, context: Union[InputContext, OutputContext]): + def _keys_from_context( + self, context: Union[InputContext, OutputContext] + ) -> Optional[Sequence[Tuple[str, ...]]]: if not context.has_asset_key: - return [None] + return None partition_keys = context.asset_partition_keys if context.has_asset_partitions else [None] return [self._get_key(context.asset_key, partition_key) for partition_key in partition_keys] @@ -134,14 +144,22 @@ def teardown_after_execution(self, context: InitResourceContext) -> None: def handle_output(self, context: OutputContext, obj: Any): keys = self._keys_from_context(context) - for key in keys: - self._values[key] = obj + if keys is None: + self._values[None] = obj + else: + for key in keys: + self._values[key] = obj def load_input(self, context: InputContext) -> Any: keys = self._keys_from_context(context) - return ( - {key[-1]: self._values[key] for key in keys} if len(keys) > 1 else self._values[keys[0]] - ) + if keys is None: + return self.values[None] + else: + return ( + {key[-1]: self._values[key] for key in keys} + if len(keys) > 1 + else self._values[keys[0]] + ) def has_value( self, asset_key: CoercibleToAssetKey, partition_key: Optional[str] = None @@ -151,9 +169,11 @@ def has_value( def get_value(self, asset_key: CoercibleToAssetKey, partition_key: Optional[str] = None) -> Any: return self._values.get(self._get_key(AssetKey.from_coercible(asset_key), partition_key)) - def _keys_from_context(self, context: Union[InputContext, OutputContext]): + def _keys_from_context( + self, context: Union[InputContext, OutputContext] + ) -> Optional[Sequence[Tuple[str, ...]]]: if not context.has_asset_key: - return [None] + return None partition_keys = context.asset_partition_keys if context.has_asset_partitions else [None] return [self._get_key(context.asset_key, partition_key) for partition_key in partition_keys] diff --git a/python_modules/dagster/setup.py b/python_modules/dagster/setup.py index f029527d1df55..4c668740b32b1 100644 --- a/python_modules/dagster/setup.py +++ b/python_modules/dagster/setup.py @@ -133,9 +133,7 @@ def get_version() -> str: "tox==3.25.0", "morefs[asynclocal]; python_version>='3.8'", ], - "mypy": [ - "mypy==0.991", - ], + "mypy": ["mypy==1.8.0"], "pyright": [ "pyright==1.1.349", ### Stub packages diff --git a/python_modules/libraries/dagster-airbyte/dagster_airbyte/managed/reconciliation.py b/python_modules/libraries/dagster-airbyte/dagster_airbyte/managed/reconciliation.py index dd48a3fb505c5..3c945c46163f2 100644 --- a/python_modules/libraries/dagster-airbyte/dagster_airbyte/managed/reconciliation.py +++ b/python_modules/libraries/dagster-airbyte/dagster_airbyte/managed/reconciliation.py @@ -571,7 +571,7 @@ def reconcile_connections_post( # Enable or disable basic normalization based on config normalization_operation_id = reconcile_normalization( res, - existing_connections.get("name", {}).get("connectionId"), + existing_connections.get("name", {}).get("connectionId"), # type: ignore # (bad stubs) destination, config_conn.normalize_data, workspace_id, diff --git a/python_modules/libraries/dagster-airbyte/dagster_airbyte/managed/types.py b/python_modules/libraries/dagster-airbyte/dagster_airbyte/managed/types.py index 91906ab91825b..ff07f72cc42a6 100644 --- a/python_modules/libraries/dagster-airbyte/dagster_airbyte/managed/types.py +++ b/python_modules/libraries/dagster-airbyte/dagster_airbyte/managed/types.py @@ -5,6 +5,7 @@ import dagster._check as check from dagster._annotations import deprecated, public +from typing_extensions import Self MANAGED_ELEMENTS_DEPRECATION_MSG = ( "Dagster is deprecating support for ingestion-as-code." @@ -303,7 +304,7 @@ def from_api_json( api_dict: Mapping[str, Any], init_sources: Mapping[str, InitializedAirbyteSource], init_dests: Mapping[str, InitializedAirbyteDestination], - ): + ) -> Self: source = next( ( source.source diff --git a/python_modules/libraries/dagster-airbyte/dagster_airbyte/resources.py b/python_modules/libraries/dagster-airbyte/dagster_airbyte/resources.py index c254afe4b0d3e..9a1ac1cf81ae9 100644 --- a/python_modules/libraries/dagster-airbyte/dagster_airbyte/resources.py +++ b/python_modules/libraries/dagster-airbyte/dagster_airbyte/resources.py @@ -478,12 +478,15 @@ def get_default_workspace(self) -> str: def get_source_definition_by_name(self, name: str) -> Optional[str]: name_lower = name.lower() - definitions = self.make_request_cached(endpoint="/source_definitions/list", data={}) + definitions = check.not_none( + self.make_request_cached(endpoint="/source_definitions/list", data={}) + ) + source_definitions = cast(List[Dict[str, Any]], definitions["sourceDefinitions"]) return next( ( definition["sourceDefinitionId"] - for definition in definitions["sourceDefinitions"] + for definition in source_definitions if definition["name"].lower() == name_lower ), None, diff --git a/python_modules/libraries/dagster-airflow/dagster_airflow_tests/test_dagster_job_factory/test_dag_run_conf.py b/python_modules/libraries/dagster-airflow/dagster_airflow_tests/test_dagster_job_factory/test_dag_run_conf.py index e30c269f8d17d..cbab7f85cf0c1 100644 --- a/python_modules/libraries/dagster-airflow/dagster_airflow_tests/test_dagster_job_factory/test_dag_run_conf.py +++ b/python_modules/libraries/dagster-airflow/dagster_airflow_tests/test_dagster_job_factory/test_dag_run_conf.py @@ -43,6 +43,7 @@ def test_dag_run_conf_local() -> None: dag_bag = DagBag(dag_folder=dags_path) retry_dag = dag_bag.get_dag(dag_id="dag_run_conf_dag") + assert retry_dag is not None, "DAG not found" job = make_dagster_job_from_airflow_dag( dag=retry_dag, resource_defs={"airflow_db": airflow_db} diff --git a/python_modules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py b/python_modules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py index c76f812a9df0e..bc3f2b029ab7a 100644 --- a/python_modules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py +++ b/python_modules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py @@ -429,7 +429,7 @@ def _grant_permissions( # Update job permissions if "job_permissions" in self.permissions: job_permissions = self._format_permissions(self.permissions["job_permissions"]) - job_id = run_info.job_id # type: ignore # (??) + job_id = run_info.job_id log.debug(f"Updating job permissions with following json: {job_permissions}") client.permissions.update("jobs", job_id, access_control_list=job_permissions) log.info("Successfully updated cluster permissions") diff --git a/python_modules/libraries/dagstermill/dagstermill/examples/repository.py b/python_modules/libraries/dagstermill/dagstermill/examples/repository.py index 00bd3ec2ef7dc..5416b8d10427c 100644 --- a/python_modules/libraries/dagstermill/dagstermill/examples/repository.py +++ b/python_modules/libraries/dagstermill/dagstermill/examples/repository.py @@ -303,12 +303,12 @@ def error_job(): if DAGSTER_PANDAS_PRESENT and SKLEARN_PRESENT and MATPLOTLIB_PRESENT: # We need type-ignores here because type checkers don't understand the `*_PRESENT` kwargs. - clean_data = test_nb_op("clean_data", outs={DEFAULT_OUTPUT: Out(DataFrame)}) # type: ignore + clean_data = test_nb_op("clean_data", outs={DEFAULT_OUTPUT: Out(DataFrame)}) # FIXME add an output to this - tutorial_LR = test_nb_op("tutorial_LR", ins={"df": In(DataFrame)}) # type: ignore + tutorial_LR = test_nb_op("tutorial_LR", ins={"df": In(DataFrame)}) - tutorial_RF = test_nb_op("tutorial_RF", ins={"df": In(DataFrame)}) # type: ignore + tutorial_RF = test_nb_op("tutorial_RF", ins={"df": In(DataFrame)}) @job(resource_defs=common_resource_defs) def tutorial_job(): diff --git a/scripts/gen_airbyte_classes.py b/scripts/gen_airbyte_classes.py index 59ddf293bea5b..6ab2c3a3c2716 100644 --- a/scripts/gen_airbyte_classes.py +++ b/scripts/gen_airbyte_classes.py @@ -60,21 +60,21 @@ class SchemaType(ABC): description: Optional[str] = None @abstractmethod - def get_check(self, name: str, scope: Optional[str] = None): + def get_check(self, name: str, scope: Optional[str] = None) -> str: """Returns the dagster._check check for this type, e.g. check.str_param(name, 'name').""" @abstractmethod def annotation( self, scope: Optional[str] = None, quote: bool = False, hide_default: bool = False - ): + ) -> str: """Returns the Python type annotation for this type, e.g. str or Union[str, int].""" @property - def const_value(self): + def const_value(self) -> object: """If this is a constant field, returns the constant value, otherwise returns None.""" return None - def add_description(self, description: str): + def add_description(self, description: str) -> None: if not description: return self.description = description.replace("\n", " ")