Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[dagster-tableau] Use Tableau translator instance in spec loader and state-backed defs #26740

Open
wants to merge 2 commits into
base: maxime/move-translator-context-to-translator-props-class-tableau
Choose a base branch
from
Open

[dagster-tableau] Use Tableau translator instance in spec loader and state-backed defs #26740

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b666549
[dagster-tableau] Use Tableau translator instance in spec loader and …
maximearmstrong Dec 27, 2024
2a268ea
Fix pyright and code
maximearmstrong Dec 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/content/integrations/tableau.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,8 @@ class MyCustomTableauTranslator(DagsterTableauTranslator):


tableau_specs = load_tableau_asset_specs(
tableau_workspace, dagster_tableau_translator=MyCustomTableauTranslator
tableau_workspace,
dagster_tableau_translator=MyCustomTableauTranslator,
)
defs = dg.Definitions(assets=[*tableau_specs], resources={"tableau": tableau_workspace})
```
Expand Down
3 changes: 2 additions & 1 deletion examples/docs_snippets/docs_snippets/integrations/tableau/customize-tableau-asset-defs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ def get_asset_spec(self, data: TableauContentData) -> dg.AssetSpec:


tableau_specs = load_tableau_asset_specs(
tableau_workspace, dagster_tableau_translator=MyCustomTableauTranslator
tableau_workspace,
dagster_tableau_translator=MyCustomTableauTranslator, # type: ignore
)
defs = dg.Definitions(assets=[*tableau_specs], resources={"tableau": tableau_workspace})
17 changes: 8 additions & 9 deletions python_modules/libraries/dagster-tableau/dagster_tableau/resources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,7 +503,7 @@ def build_defs(

resource_key = "tableau"

asset_specs = load_tableau_asset_specs(self, dagster_tableau_translator)
asset_specs = load_tableau_asset_specs(self, dagster_tableau_translator())

non_executable_asset_specs = [
spec
Expand Down Expand Up @@ -533,14 +533,15 @@ def build_defs(
@experimental
def load_tableau_asset_specs(
workspace: BaseTableauWorkspace,
dagster_tableau_translator: Type[DagsterTableauTranslator] = DagsterTableauTranslator,
dagster_tableau_translator: Optional[DagsterTableauTranslator] = None,
) -> Sequence[AssetSpec]:
"""Returns a list of AssetSpecs representing the Tableau content in the workspace.

Args:
workspace (Union[TableauCloudWorkspace, TableauServerWorkspace]): The Tableau workspace to fetch assets from.
dagster_tableau_translator (Type[DagsterTableauTranslator]): The translator to use
to convert Tableau content into AssetSpecs. Defaults to DagsterTableauTranslator.
dagster_tableau_translator (Optional[DagsterTableauTranslator]): The translator to use
to convert Tableau content into :py:class:`dagster.AssetSpec`.
Defaults to :py:class:`DagsterTableauTranslator`.

Returns:
List[AssetSpec]: The set of assets representing the Tableau content in the workspace.
Expand All @@ -549,7 +550,7 @@ def load_tableau_asset_specs(
return check.is_list(
TableauWorkspaceDefsLoader(
workspace=initialized_workspace,
translator_cls=dagster_tableau_translator,
translator=dagster_tableau_translator or DagsterTableauTranslator(),
)
.build_defs()
.assets,
Expand Down Expand Up @@ -598,7 +599,7 @@ def build_client(self) -> None:
@record
class TableauWorkspaceDefsLoader(StateBackedDefinitionsLoader[Mapping[str, Any]]):
workspace: BaseTableauWorkspace
translator_cls: Type[DagsterTableauTranslator]
translator: DagsterTableauTranslator

@property
def defs_key(self) -> str:
Expand All @@ -608,16 +609,14 @@ def fetch_state(self) -> TableauWorkspaceData:
return self.workspace.fetch_tableau_workspace_data()

def defs_from_state(self, state: TableauWorkspaceData) -> Definitions:
translator = self.translator_cls()

all_external_data = [
*state.data_sources_by_id.values(),
*state.sheets_by_id.values(),
*state.dashboards_by_id.values(),
]

all_external_asset_specs = [
translator.get_asset_spec(
self.translator.get_asset_spec(
TableauTranslatorData(content_data=content, workspace_data=state)
)
for content in all_external_data
Expand Down
2 changes: 1 addition & 1 deletion python_modules/libraries/dagster-tableau/dagster_tableau_tests/test_asset_specs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ def test_translator_custom_metadata(
resource.build_client()

all_asset_specs = load_tableau_asset_specs(
workspace=resource, dagster_tableau_translator=MyCustomTranslator
workspace=resource, dagster_tableau_translator=MyCustomTranslator()
)
asset_spec = next(spec for spec in all_asset_specs)

Expand Down
6 changes: 3 additions & 3 deletions python_modules/libraries/dagster-tableau/dagster_tableau_tests/test_reconstruction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from dagster_tableau.asset_utils import parse_tableau_external_and_materializable_asset_specs
from dagster_tableau.assets import build_tableau_materializable_assets_definition
from dagster_tableau.resources import TableauCloudWorkspace, load_tableau_asset_specs
from dagster_tableau.translator import DagsterTableauTranslator
from dagster_tableau.translator import DagsterTableauTranslator, TableauTranslatorData

from dagster_tableau_tests.conftest import (
FAKE_CONNECTED_APP_CLIENT_ID,
Expand Down Expand Up @@ -67,12 +67,12 @@ def cacheable_asset_defs_refreshable_workbooks():
@lazy_definitions
def cacheable_asset_defs_custom_translator():
class MyCoolTranslator(DagsterTableauTranslator):
def get_asset_spec(self, data) -> AssetSpec:
def get_asset_spec(self, data: TableauTranslatorData) -> AssetSpec:
default_spec = super().get_asset_spec(data)
return default_spec.replace_attributes(key=default_spec.key.with_prefix("my_prefix"))

tableau_specs = load_tableau_asset_specs(
workspace=resource, dagster_tableau_translator=MyCoolTranslator
workspace=resource, dagster_tableau_translator=MyCoolTranslator()
)

return Definitions(assets=[*tableau_specs], jobs=[define_asset_job("all_asset_job")])
Expand Down
Loading