From 09db0a6ed24c0922e01075d5b5555316a298862d Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Wed, 11 Oct 2023 15:04:26 +0100 Subject: [PATCH 1/5] Rename DataSet to Dataset or add Dataset where missing. Signed-off-by: Merel Theisen --- package/kedro_viz/api/graphql/serializers.py | 2 +- package/kedro_viz/api/rest/responses.py | 4 +-- .../integrations/kedro/data_loader.py | 24 ++++++++++++++++ .../kedro_viz/models/experiment_tracking.py | 6 +++- package/kedro_viz/models/flowchart.py | 18 ++++++++---- package/kedro_viz/models/utils.py | 16 +++++------ package/test_requirements.txt | 4 +-- package/tests/conftest.py | 15 +++++----- package/tests/test_api/main | 4 +-- package/tests/test_api/test_apps.py | 2 +- .../tests/test_api/test_graphql/conftest.py | 22 +++++++-------- .../test_api/test_graphql/test_queries.py | 14 +++++----- .../test_api/test_rest/test_responses.py | 18 ++++++------ .../tests/test_data_access/test_managers.py | 14 +++++----- .../test_repositories/test_data_catalog.py | 12 ++++---- .../test_modular_pipelines.py | 6 ++-- package/tests/test_models/test_flowchart.py | 28 +++++++++---------- 17 files changed, 121 insertions(+), 88 deletions(-) diff --git a/package/kedro_viz/api/graphql/serializers.py b/package/kedro_viz/api/graphql/serializers.py index 41d1fe87e..adc560f2c 100644 --- a/package/kedro_viz/api/graphql/serializers.py +++ b/package/kedro_viz/api/graphql/serializers.py @@ -86,7 +86,7 @@ def format_run_tracking_data( Dictionary with formatted tracking data for selected runs Example: - >>> from kedro_datasets.tracking import MetricsDataSet + >>> from kedro_datasets.tracking import MetricsDataset >>> tracking_data = { >>> 'My Favorite Sprint': { >>> 'bootstrap':0.8 diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 7b434577c..bb49e919c 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -95,7 +95,7 @@ class Config: "modular_pipelines": [], "type": "data", "layer": "primary", - "dataset_type": "kedro.extras.datasets.pandas.csv_dataset.CSVDataSet", + "dataset_type": "kedro_datasets.pandas.csv_dataset.CSVDataset", "stats": {"rows": 10, "columns": 2, "file_size": 2300}, } } @@ -142,7 +142,7 @@ class Config: schema_extra = { "example": { "filepath": "/my-kedro-project/data/03_primary/master_table.csv", - "type": "pandas.csv_dataset.CSVDataSet", + "type": "kedro_datasets.pandas.csv_dataset.CSVDataset", "run_command": "kedro run --to-outputs=master_table", } } diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index ca4b1565b..812d720b8 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -181,20 +181,44 @@ def matplotlib_writer_load(dataset: matplotlib.MatplotlibWriter) -> str: except (ImportError, AttributeError): pass +try: + getattr(plotly, "JSONDataset") # Trigger import + plotly.JSONDataset._load = json_dataset.JSONDataset._load +except (ImportError, AttributeError): + pass + try: getattr(plotly, "PlotlyDataSet") # Trigger import plotly.PlotlyDataSet._load = json_dataset.JSONDataSet._load except (ImportError, AttributeError): pass +try: + getattr(plotly, "PlotlyDataset") # Trigger import + plotly.PlotlyDataset._load = json_dataset.JSONDataset._load +except (ImportError, AttributeError): + pass + try: getattr(tracking, "JSONDataSet") # Trigger import tracking.JSONDataSet._load = json_dataset.JSONDataSet._load except (ImportError, AttributeError): pass +try: + getattr(tracking, "JSONDataset") # Trigger import + tracking.JSONDataset._load = json_dataset.JSONDataset._load +except (ImportError, AttributeError): + pass + try: getattr(tracking, "MetricsDataSet") # Trigger import tracking.MetricsDataSet._load = json_dataset.JSONDataSet._load except (ImportError, AttributeError): pass + +try: + getattr(tracking, "MetricsDataset") # Trigger import + tracking.MetricsDataset._load = json_dataset.JSONDataset._load +except (ImportError, AttributeError): + pass diff --git a/package/kedro_viz/models/experiment_tracking.py b/package/kedro_viz/models/experiment_tracking.py index ce8cf012c..63debe17e 100644 --- a/package/kedro_viz/models/experiment_tracking.py +++ b/package/kedro_viz/models/experiment_tracking.py @@ -64,10 +64,14 @@ class TrackingDatasetGroup(str, Enum): # Map dataset types (as produced by get_dataset_type) to their group TRACKING_DATASET_GROUPS = { "plotly.plotly_dataset.PlotlyDataSet": TrackingDatasetGroup.PLOT, + "plotly.plotly_dataset.PlotlyDataset": TrackingDatasetGroup.PLOT, "plotly.json_dataset.JSONDataSet": TrackingDatasetGroup.PLOT, + "plotly.json_dataset.JSONDataset": TrackingDatasetGroup.PLOT, "matplotlib.matplotlib_writer.MatplotlibWriter": TrackingDatasetGroup.PLOT, "tracking.metrics_dataset.MetricsDataSet": TrackingDatasetGroup.METRIC, + "tracking.metrics_dataset.MetricsDataset": TrackingDatasetGroup.METRIC, "tracking.json_dataset.JSONDataSet": TrackingDatasetGroup.JSON, + "tracking.json_dataset.JSONDataset": TrackingDatasetGroup.JSON, } @@ -77,7 +81,7 @@ class TrackingDatasetModel: dataset_name: str # dataset is the actual dataset instance, whereas dataset_type is a string. - # e.g. "tracking.metrics_dataset.MetricsDataSet" + # e.g. "tracking.metrics_dataset.MetricsDataset" dataset: "AbstractVersionedDataset" dataset_type: str = field(init=False) # runs is a mapping from run_id to loaded data. diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index a59d11f4c..0a0be5e02 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -181,7 +181,7 @@ def create_data_node( stats: Optional[Dict], is_free_input: bool = False, ) -> Union["DataNode", "TranscodedDataNode"]: - """Create a graph node of type DATA for a given Kedro DataSet instance. + """Create a graph node of type DATA for a given Kedro Dataset instance. Args: dataset_name: The name of the dataset, including namespace, e.g. data_science.master_table. @@ -465,7 +465,9 @@ def is_plot_node(self): """ return self.dataset_type in ( "plotly.plotly_dataset.PlotlyDataSet", + "plotly.plotly_dataset.PlotlyDataset", "plotly.json_dataset.JSONDataSet", + "plotly.json_dataset.JSONDataset", ) def is_image_node(self): @@ -474,11 +476,15 @@ def is_image_node(self): def is_metric_node(self): """Check if the current node is a metrics node.""" - return self.dataset_type == "tracking.metrics_dataset.MetricsDataSet" + return self.dataset_type in ( + "tracking.metrics_dataset.MetricsDataSet", + "tracking.metrics_dataset.MetricsDataset") def is_json_node(self): - """Check if the current node is a JSONDataSet node.""" - return self.dataset_type == "tracking.json_dataset.JSONDataSet" + """Check if the current node is a JSONDataset node.""" + return self.dataset_type in ( + "tracking.json_dataset.JSONDataSet", + "tracking.json_dataset.JSONDataset") def is_tracking_node(self): """Checks if the current node is a tracking data node""" @@ -543,7 +549,7 @@ def __post_init__(self): class DataNodeMetadata(GraphNodeMetadata): """Represent the metadata of a DataNode""" - # the dataset type for this data node, e.g. CSVDataSet + # the dataset type for this data node, e.g. CSVDataset type: Optional[str] = field(init=False) # the path to the actual data file for the underlying dataset. @@ -554,7 +560,7 @@ class DataNodeMetadata(GraphNodeMetadata): data_node: InitVar[DataNode] # the optional plot data if the underlying dataset has a plot. - # currently only applicable for PlotlyDataSet + # currently only applicable for PlotlyDataset plot: Optional[Dict] = field(init=False, default=None) # the optional image data if the underlying dataset has a image. diff --git a/package/kedro_viz/models/utils.py b/package/kedro_viz/models/utils.py index aca687a39..7a5057bd2 100644 --- a/package/kedro_viz/models/utils.py +++ b/package/kedro_viz/models/utils.py @@ -19,17 +19,17 @@ def get_dataset_type(dataset: "AbstractDataset") -> str: which ``dataset`` belongs, joined with the name of its class. :: - >>> get_dataset_type(kedro.extras.datasets.plotly.plotly_dataset.PlotlyDataSet()) - plotly.plotly_dataset.PlotlyDataSet + >>> get_dataset_type(kedro.extras.datasets.plotly.plotly_dataset.PlotlyDataset()) + plotly.plotly_dataset.PlotlyDataset - >>> get_dataset_type(kedro_datasets.plotly.plotly_dataset.PlotlyDataSet()) - plotly.plotly_dataset.PlotlyDataSet + >>> get_dataset_type(kedro_datasets.plotly.plotly_dataset.PlotlyDataset()) + plotly.plotly_dataset.PlotlyDataset - >>> get_dataset_type(my.custom.path.to.plotly.plotly_dataset.PlotlyDataSet()) - plotly.plotly_dataset.PlotlyDataSet + >>> get_dataset_type(my.custom.path.to.plotly.plotly_dataset.PlotlyDataset()) + plotly.plotly_dataset.PlotlyDataset - >>> get_dataset_type(package.PlotlyDataSet()) - package.PlotlyDataSet + >>> get_dataset_type(package.PlotlyDataset()) + package.PlotlyDataset Args: dataset: The dataset object to get the type of diff --git a/package/test_requirements.txt b/package/test_requirements.txt index eefc42fab..0c9877738 100644 --- a/package/test_requirements.txt +++ b/package/test_requirements.txt @@ -1,7 +1,7 @@ -r requirements.txt kedro >=0.17.0 -kedro-datasets[pandas.ParquetDataSet, pandas.CSVDataSet, pandas.ExcelDataSet, plotly.JSONDataSet]>=1.0, <1.7.1 +kedro-datasets[pandas.ParquetDataset, pandas.CSVDataset, pandas.ExcelDataset, plotly.JSONDataset]~=1.0 kedro-telemetry>=0.1.1 # for testing telemetry integration bandit~=1.7 behave~=1.2 @@ -12,7 +12,7 @@ fastapi[all]>=0.73.0, <0.96.0 isort~=5.11 matplotlib~=3.5 mypy~=1.0 -moto~=4.1.14 +moto~=4.1.14 psutil==5.9.5 # same as Kedro for now pylint~=2.17 pytest~=7.4 diff --git a/package/tests/conftest.py b/package/tests/conftest.py index 95100a8ec..238f15992 100644 --- a/package/tests/conftest.py +++ b/package/tests/conftest.py @@ -22,7 +22,6 @@ # kedro 0.18.11 onwards from kedro.io import MemoryDataset except ImportError: - # older versions # older versions from kedro.io import MemoryDataSet as MemoryDataset @@ -97,8 +96,8 @@ def train_model(model_inputs, parameters): def example_catalog(): yield DataCatalog( data_sets={ - "uk.data_processing.raw_data": pandas.CSVDataSet(filepath="raw_data.csv"), - "model_inputs": pandas.CSVDataSet(filepath="model_inputs.csv"), + "uk.data_processing.raw_data": pandas.CSVDataset(filepath="raw_data.csv"), + "model_inputs": pandas.CSVDataset(filepath="model_inputs.csv"), "uk.data_science.model": MemoryDataset(), }, feed_dict={ @@ -151,10 +150,10 @@ def train_model(model_inputs, parameters): def example_transcoded_catalog(): yield DataCatalog( data_sets={ - "model_inputs@pandas": pandas.ParquetDataSet( + "model_inputs@pandas": pandas.ParquetDataset( filepath="model_inputs.parquet" ), - "model_inputs@pandas2": pandas.CSVDataSet(filepath="model_inputs.csv"), + "model_inputs@pandas2": pandas.CSVDataset(filepath="model_inputs.csv"), }, feed_dict={ "parameters": {"train_test_split": 0.1, "num_epochs": 1000}, @@ -234,12 +233,12 @@ def example_run_ids(): @pytest.fixture def example_multiple_run_tracking_dataset(example_run_ids, tmp_path): - new_metrics_dataset = tracking.MetricsDataSet( + new_metrics_dataset = tracking.MetricsDataset( filepath=Path(tmp_path / "test.json").as_posix(), version=Version(None, example_run_ids[1]), ) new_metrics_dataset.save({"col1": 1, "col3": 3}) - new_metrics_dataset = tracking.MetricsDataSet( + new_metrics_dataset = tracking.MetricsDataset( filepath=Path(tmp_path / "test.json").as_posix(), version=Version(None, example_run_ids[0]), ) @@ -286,7 +285,7 @@ def example_dataset_stats_hook_obj(): @pytest.fixture def example_csv_dataset(tmp_path, example_data_frame): - new_csv_dataset = pandas.CSVDataSet( + new_csv_dataset = pandas.CSVDataset( filepath=Path(tmp_path / "model_inputs.csv").as_posix(), ) new_csv_dataset.save(example_data_frame) diff --git a/package/tests/test_api/main b/package/tests/test_api/main index 1ffb622b1..47d66a304 100644 --- a/package/tests/test_api/main +++ b/package/tests/test_api/main @@ -27,7 +27,7 @@ "type": "data", "modular_pipelines": ["uk", "uk.data_processing"], "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "0ecea0de", @@ -37,7 +37,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_inputs", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "7b140b3f", diff --git a/package/tests/test_api/test_apps.py b/package/tests/test_api/test_apps.py index d76e8e5f4..ac64bb531 100644 --- a/package/tests/test_api/test_apps.py +++ b/package/tests/test_api/test_apps.py @@ -82,7 +82,7 @@ class TestNodeMetadataEndpoint: ( "13399a82", 200, - {"filepath": "raw_data.csv", "type": "pandas.csv_dataset.CSVDataSet"}, + {"filepath": "raw_data.csv", "type": "pandas.csv_dataset.CSVDataset"}, ), ], ) diff --git a/package/tests/test_api/test_graphql/conftest.py b/package/tests/test_api/test_graphql/conftest.py index d2df71cca..007a134ba 100644 --- a/package/tests/test_api/test_graphql/conftest.py +++ b/package/tests/test_api/test_graphql/conftest.py @@ -88,30 +88,30 @@ def save_version(example_run_ids): @pytest.fixture def example_tracking_catalog(example_run_ids, tmp_path): example_run_id = example_run_ids[0] - metrics_dataset = tracking.MetricsDataSet( + metrics_dataset = tracking.MetricsDataset( filepath=Path(tmp_path / "test.json").as_posix(), version=Version(None, example_run_id), ) metrics_dataset.save({"col1": 1, "col2": 2, "col3": 3}) - csv_dataset = pandas.CSVDataSet( + csv_dataset = pandas.CSVDataset( Path(tmp_path / "metrics.csv").as_posix(), version=Version(None, example_run_id), ) - more_metrics = tracking.MetricsDataSet( + more_metrics = tracking.MetricsDataset( filepath=Path(tmp_path / "metrics.json").as_posix(), version=Version(None, example_run_id), ) more_metrics.save({"col4": 4, "col5": 5, "col6": 6}) - json_dataset = tracking.JSONDataSet( + json_dataset = tracking.JSONDataset( filepath=Path(tmp_path / "tracking.json").as_posix(), version=Version(None, example_run_id), ) json_dataset.save({"col7": "column_seven", "col2": True, "col3": 3}) - plotly_dataset = plotly.JSONDataSet( + plotly_dataset = plotly.JSONDataset( filepath=Path(tmp_path / "plotly.json").as_posix(), version=Version(None, example_run_id), ) @@ -168,12 +168,12 @@ def savefig(cls, bytes_buffer, **kwargs): @pytest.fixture def example_multiple_run_tracking_catalog(example_run_ids, tmp_path): - new_metrics_dataset = tracking.MetricsDataSet( + new_metrics_dataset = tracking.MetricsDataset( filepath=Path(tmp_path / "test.json").as_posix(), version=Version(None, example_run_ids[1]), ) new_metrics_dataset.save({"col1": 1, "col3": 3}) - new_metrics_dataset = tracking.MetricsDataSet( + new_metrics_dataset = tracking.MetricsDataset( filepath=Path(tmp_path / "test.json").as_posix(), version=Version(None, example_run_ids[0]), ) @@ -192,12 +192,12 @@ def example_multiple_run_tracking_catalog(example_run_ids, tmp_path): def example_multiple_run_tracking_catalog_at_least_one_empty_run( example_run_ids, tmp_path ): - new_metrics_dataset = tracking.MetricsDataSet( + new_metrics_dataset = tracking.MetricsDataset( filepath=Path(tmp_path / "test.json").as_posix(), version=Version(None, example_run_ids[1]), ) new_metrics_dataset.save({"col1": 1, "col3": 3}) - new_metrics_dataset = tracking.MetricsDataSet( + new_metrics_dataset = tracking.MetricsDataset( filepath=Path(tmp_path / "test.json").as_posix(), version=Version(None, example_run_ids[0]), ) @@ -212,11 +212,11 @@ def example_multiple_run_tracking_catalog_at_least_one_empty_run( @pytest.fixture def example_multiple_run_tracking_catalog_all_empty_runs(example_run_ids, tmp_path): - new_metrics_dataset = tracking.MetricsDataSet( + new_metrics_dataset = tracking.MetricsDataset( filepath=Path(tmp_path / "test.json").as_posix(), version=Version(None, example_run_ids[1]), ) - new_metrics_dataset = tracking.MetricsDataSet( + new_metrics_dataset = tracking.MetricsDataset( filepath=Path(tmp_path / "test.json").as_posix(), version=Version(None, example_run_ids[0]), ) diff --git a/package/tests/test_api/test_graphql/test_queries.py b/package/tests/test_api/test_graphql/test_queries.py index 29a71d383..f5d11ba5d 100644 --- a/package/tests/test_api/test_graphql/test_queries.py +++ b/package/tests/test_api/test_graphql/test_queries.py @@ -93,7 +93,7 @@ def test_run_tracking_data_query( "metrics": [ { "datasetName": "metrics", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "data": { "col1": [{"runId": example_run_id, "value": 1.0}], "col2": [{"runId": example_run_id, "value": 2.0}], @@ -102,7 +102,7 @@ def test_run_tracking_data_query( }, { "datasetName": "more_metrics", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "data": { "col4": [{"runId": example_run_id, "value": 4.0}], "col5": [{"runId": example_run_id, "value": 5.0}], @@ -113,7 +113,7 @@ def test_run_tracking_data_query( "json": [ { "datasetName": "json_tracking", - "datasetType": "tracking.json_dataset.JSONDataSet", + "datasetType": "tracking.json_dataset.JSONDataset", "data": { "col2": [{"runId": example_run_id, "value": True}], "col3": [{"runId": example_run_id, "value": 3}], @@ -129,7 +129,7 @@ def test_run_tracking_data_query( "plots": [ { "datasetName": "plotly_dataset", - "datasetType": "plotly.json_dataset.JSONDataSet", + "datasetType": "plotly.json_dataset.JSONDataset", "data": { "plotly.json": [ { @@ -224,7 +224,7 @@ def test_metrics_data( "runTrackingData": [ { "datasetName": "new_metrics", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "data": { "col1": [ { @@ -261,7 +261,7 @@ def test_metrics_data( "runTrackingData": [ { "datasetName": "new_metrics", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "data": { "col1": [ { @@ -312,7 +312,7 @@ def test_graphql_run_tracking_data( "runTrackingData": [ { "datasetName": "new_metrics", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "data": { "col1": [ { diff --git a/package/tests/test_api/test_rest/test_responses.py b/package/tests/test_api/test_rest/test_responses.py index 2dacbc970..7adbd1964 100644 --- a/package/tests/test_api/test_rest/test_responses.py +++ b/package/tests/test_api/test_rest/test_responses.py @@ -119,7 +119,7 @@ def assert_example_data(response_data): "modular_pipelines": ["uk", "uk.data_processing"], "type": "data", "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet", + "dataset_type": "pandas.csv_dataset.CSVDataset", "stats": None, }, { @@ -141,7 +141,7 @@ def assert_example_data(response_data): "modular_pipelines": [], "type": "data", "layer": "model_inputs", - "dataset_type": "pandas.csv_dataset.CSVDataSet", + "dataset_type": "pandas.csv_dataset.CSVDataset", "stats": {"columns": 12, "rows": 29768}, }, { @@ -318,7 +318,7 @@ def assert_example_data_from_file(response_data): "modular_pipelines": ["uk", "uk.data_processing"], "type": "data", "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet", + "dataset_type": "pandas.csv_dataset.CSVDataset", }, { "id": "f0ebef01", @@ -338,7 +338,7 @@ def assert_example_data_from_file(response_data): "modular_pipelines": [], "type": "data", "layer": "model_inputs", - "dataset_type": "pandas.csv_dataset.CSVDataSet", + "dataset_type": "pandas.csv_dataset.CSVDataset", }, { "id": "7b140b3f", @@ -592,9 +592,9 @@ def test_transcoded_data_node_metadata(self, example_transcoded_api): response = client.get("/api/nodes/0ecea0de") assert response.json() == { "filepath": "model_inputs.csv", - "original_type": "pandas.csv_dataset.CSVDataSet", + "original_type": "pandas.csv_dataset.CSVDataset", "transcoded_types": [ - "pandas.parquet_dataset.ParquetDataSet", + "pandas.parquet_dataset.ParquetDataset", ], "run_command": "kedro run --to-outputs=model_inputs@pandas2", } @@ -628,7 +628,7 @@ def test_data_node_metadata(self, client): response = client.get("/api/nodes/0ecea0de") assert response.json() == { "filepath": "model_inputs.csv", - "type": "pandas.csv_dataset.CSVDataSet", + "type": "pandas.csv_dataset.CSVDataset", "run_command": "kedro run --to-outputs=model_inputs", "stats": {"columns": 12, "rows": 29768}, } @@ -637,7 +637,7 @@ def test_data_node_metadata_for_free_input(self, client): response = client.get("/api/nodes/13399a82") assert response.json() == { "filepath": "raw_data.csv", - "type": "pandas.csv_dataset.CSVDataSet", + "type": "pandas.csv_dataset.CSVDataset", } def test_parameters_node_metadata(self, client): @@ -686,7 +686,7 @@ def test_get_pipeline(self, client): "modular_pipelines": [], "type": "data", "layer": "model_inputs", - "dataset_type": "pandas.csv_dataset.CSVDataSet", + "dataset_type": "pandas.csv_dataset.CSVDataset", "stats": {"columns": 12, "rows": 29768}, }, { diff --git a/package/tests/test_data_access/test_managers.py b/package/tests/test_data_access/test_managers.py index e21b3d02f..4c6d16c2e 100644 --- a/package/tests/test_data_access/test_managers.py +++ b/package/tests/test_data_access/test_managers.py @@ -5,7 +5,7 @@ from kedro.io import DataCatalog from kedro.pipeline import Pipeline, node from kedro.pipeline.modular_pipeline import pipeline -from kedro_datasets.pandas import CSVDataSet +from kedro_datasets.pandas import CSVDataset from kedro_viz.constants import DEFAULT_REGISTERED_PIPELINE_ID, ROOT_MODULAR_PIPELINE_ID from kedro_viz.data_access.managers import DataAccessManager @@ -32,7 +32,7 @@ def identity(x): class TestAddCatalog: def test_add_catalog(self, data_access_manager: DataAccessManager): - dataset = CSVDataSet(filepath="dataset.csv") + dataset = CSVDataset(filepath="dataset.csv") catalog = DataCatalog(data_sets={"dataset": dataset}) data_access_manager.add_catalog(catalog) assert data_access_manager.catalog.get_catalog() is catalog @@ -73,7 +73,7 @@ def test_add_node_with_modular_pipeline( ] def test_add_node_input(self, data_access_manager: DataAccessManager): - dataset = CSVDataSet(filepath="dataset.csv") + dataset = CSVDataset(filepath="dataset.csv") dataset_name = "x" registered_pipeline_id = "my_pipeline" @@ -167,7 +167,7 @@ def test_parameters_yaml_namespace_not_added_to_modular_pipelines( assert "uk.data_science.train_test_split" not in modular_pipelines_tree def test_add_node_output(self, data_access_manager: DataAccessManager): - dataset = CSVDataSet(filepath="dataset.csv") + dataset = CSVDataset(filepath="dataset.csv") registered_pipeline_id = "my_pipeline" dataset_name = "x" @@ -205,9 +205,9 @@ def test_add_node_output(self, data_access_manager: DataAccessManager): } -class TestAddDataSet: +class TestAddDataset: def test_add_dataset(self, data_access_manager: DataAccessManager): - dataset = CSVDataSet(filepath="dataset.csv") + dataset = CSVDataset(filepath="dataset.csv") dataset_name = "x" catalog = DataCatalog( data_sets={dataset_name: dataset}, @@ -242,7 +242,7 @@ def test_add_memory_dataset_when_dataset_not_in_catalog( def test_add_dataset_with_modular_pipeline( self, data_access_manager: DataAccessManager ): - dataset = CSVDataSet(filepath="dataset.csv") + dataset = CSVDataset(filepath="dataset.csv") dataset_name = "uk.data_science.x" catalog = DataCatalog( data_sets={dataset_name: dataset}, diff --git a/package/tests/test_data_access/test_repositories/test_data_catalog.py b/package/tests/test_data_access/test_repositories/test_data_catalog.py index 83efe7075..bbbd3f54d 100644 --- a/package/tests/test_data_access/test_repositories/test_data_catalog.py +++ b/package/tests/test_data_access/test_repositories/test_data_catalog.py @@ -9,7 +9,7 @@ def test_get_layer_mapping_for_transcoded_dataset(self): repo = CatalogRepository() catalog_config = { "cars@pandas": { - "type": "pandas.CSVDataSet", + "type": "pandas.CSVDataset", "filepath": "cars.csv", "layer": "raw", }, @@ -23,7 +23,7 @@ def test_validate_layers_error(self): repo = CatalogRepository() catalog_config = { "car@pandas1": { - "type": "pandas.CSVDataSet", + "type": "pandas.CSVDataset", "filepath": "cars.csv", "metadata": { "kedro-viz": { @@ -32,7 +32,7 @@ def test_validate_layers_error(self): }, }, "car@pandas2": { - "type": "pandas.ParquetDataSet", + "type": "pandas.ParquetDataset", "filepath": "cars.pq", "metadata": { "kedro-viz": { @@ -55,7 +55,7 @@ def test_get_layer_mapping_from_metadata(self): repo = CatalogRepository() catalog_config = { "car@pandas1": { - "type": "pandas.CSVDataSet", + "type": "pandas.CSVDataset", "filepath": "cars.csv", "metadata": { "kedro-viz": { @@ -73,7 +73,7 @@ def test_get_layer_mapping_from_metadata_and_top_level_layer(self): repo = CatalogRepository() catalog_config = { "car_1": { - "type": "pandas.CSVDataSet", + "type": "pandas.CSVDataset", "filepath": "cars.csv", "metadata": { "kedro-viz": { @@ -82,7 +82,7 @@ def test_get_layer_mapping_from_metadata_and_top_level_layer(self): }, }, "car_2": { - "type": "pandas.CSVDataSet", + "type": "pandas.CSVDataset", "filepath": "cars.csv", "layer": "raw", }, diff --git a/package/tests/test_data_access/test_repositories/test_modular_pipelines.py b/package/tests/test_data_access/test_repositories/test_modular_pipelines.py index 97a4fe0f4..fe37a6093 100644 --- a/package/tests/test_data_access/test_repositories/test_modular_pipelines.py +++ b/package/tests/test_data_access/test_repositories/test_modular_pipelines.py @@ -1,6 +1,6 @@ import pytest from kedro.pipeline import node -from kedro_datasets.pandas import CSVDataSet +from kedro_datasets.pandas import CSVDataset from kedro_viz.constants import ROOT_MODULAR_PIPELINE_ID from kedro_viz.data_access.repositories import ModularPipelinesRepository @@ -37,7 +37,7 @@ def test_extract_from_node(self, identity): assert modular_pipelines.has_modular_pipeline("data_science") def test_add_input(self): - kedro_dataset = CSVDataSet(filepath="foo.csv") + kedro_dataset = CSVDataset(filepath="foo.csv") modular_pipelines = ModularPipelinesRepository() data_science_pipeline = modular_pipelines.get_or_create_modular_pipeline( "data_science" @@ -53,7 +53,7 @@ def test_add_input(self): assert data_node.id in data_science_pipeline.inputs def test_add_output(self): - kedro_dataset = CSVDataSet(filepath="foo.csv") + kedro_dataset = CSVDataset(filepath="foo.csv") modular_pipelines = ModularPipelinesRepository() data_science_pipeline = modular_pipelines.get_or_create_modular_pipeline( "data_science" diff --git a/package/tests/test_models/test_flowchart.py b/package/tests/test_models/test_flowchart.py index 0caa179e1..0ac1d6281 100644 --- a/package/tests/test_models/test_flowchart.py +++ b/package/tests/test_models/test_flowchart.py @@ -7,7 +7,7 @@ import pytest from kedro.pipeline.node import node -from kedro_datasets.pandas import CSVDataSet, ParquetDataSet +from kedro_datasets.pandas import CSVDataset, ParquetDataset from kedro_viz.models.flowchart import ( DataNode, @@ -112,7 +112,7 @@ def test_create_task_node(self, namespace, expected_modular_pipelines): ], ) def test_create_data_node(self, dataset_name, expected_modular_pipelines): - kedro_dataset = CSVDataSet(filepath="foo.csv") + kedro_dataset = CSVDataset(filepath="foo.csv") data_node = GraphNode.create_data_node( dataset_name=dataset_name, layer="raw", @@ -148,7 +148,7 @@ def test_create_data_node(self, dataset_name, expected_modular_pipelines): ], ) def test_create_transcoded_data_node(self, transcoded_dataset_name, original_name): - kedro_dataset = CSVDataSet(filepath="foo.csv") + kedro_dataset = CSVDataset(filepath="foo.csv") data_node = GraphNode.create_data_node( dataset_name=transcoded_dataset_name, layer="raw", @@ -254,7 +254,7 @@ def test_modular_pipeline_name(self): def test_add_node_to_pipeline(self): default_pipeline = RegisteredPipeline("__default__") another_pipeline = RegisteredPipeline("testing") - kedro_dataset = CSVDataSet(filepath="foo.csv") + kedro_dataset = CSVDataset(filepath="foo.csv") data_node = GraphNode.create_data_node( dataset_name="dataset@transcoded", layer="raw", @@ -357,7 +357,7 @@ def test_task_node_metadata_with_partial_func(self): assert task_node_metadata.outputs == ["y"] def test_data_node_metadata(self): - dataset = CSVDataSet(filepath="/tmp/dataset.csv") + dataset = CSVDataset(filepath="/tmp/dataset.csv") data_node = GraphNode.create_data_node( dataset_name="dataset", layer="raw", @@ -366,7 +366,7 @@ def test_data_node_metadata(self): stats={"rows": 10, "columns": 2}, ) data_node_metadata = DataNodeMetadata(data_node=data_node) - assert data_node_metadata.type == "pandas.csv_dataset.CSVDataSet" + assert data_node_metadata.type == "pandas.csv_dataset.CSVDataset" assert data_node_metadata.filepath == "/tmp/dataset.csv" assert data_node_metadata.run_command == "kedro run --to-outputs=dataset" assert data_node_metadata.stats["rows"] == 10 @@ -374,7 +374,7 @@ def test_data_node_metadata(self): def test_preview_args_not_exist(self): metadata = {"kedro-viz": {"something": 3}} - dataset = CSVDataSet(filepath="test.csv", metadata=metadata) + dataset = CSVDataset(filepath="test.csv", metadata=metadata) data_node = GraphNode.create_data_node( dataset_name="dataset", tags=set(), layer=None, dataset=dataset, stats=None ) @@ -382,7 +382,7 @@ def test_preview_args_not_exist(self): def test_get_preview_args(self): metadata = {"kedro-viz": {"preview_args": {"nrows": 3}}} - dataset = CSVDataSet(filepath="test.csv", metadata=metadata) + dataset = CSVDataset(filepath="test.csv", metadata=metadata) data_node = GraphNode.create_data_node( dataset_name="dataset", tags=set(), layer=None, dataset=dataset, stats=None ) @@ -422,7 +422,7 @@ def test_preview_data_node_metadata_not_exist(self): assert preview_node_metadata.plot is None def test_transcoded_data_node_metadata(self): - dataset = CSVDataSet(filepath="/tmp/dataset.csv") + dataset = CSVDataset(filepath="/tmp/dataset.csv") transcoded_data_node = GraphNode.create_data_node( dataset_name="dataset@pandas2", layer="raw", @@ -431,24 +431,24 @@ def test_transcoded_data_node_metadata(self): stats={"rows": 10, "columns": 2}, ) transcoded_data_node.original_name = "dataset" - transcoded_data_node.original_version = ParquetDataSet(filepath="foo.parquet") - transcoded_data_node.transcoded_versions = [CSVDataSet(filepath="foo.csv")] + transcoded_data_node.original_version = ParquetDataset(filepath="foo.parquet") + transcoded_data_node.transcoded_versions = [CSVDataset(filepath="foo.csv")] transcoded_data_node_metadata = TranscodedDataNodeMetadata( transcoded_data_node=transcoded_data_node ) assert ( transcoded_data_node_metadata.original_type - == "pandas.parquet_dataset.ParquetDataSet" + == "pandas.parquet_dataset.ParquetDataset" ) assert transcoded_data_node_metadata.transcoded_types == [ - "pandas.csv_dataset.CSVDataSet" + "pandas.csv_dataset.CSVDataset" ] assert transcoded_data_node_metadata.stats["rows"] == 10 assert transcoded_data_node_metadata.stats["columns"] == 2 def test_partitioned_data_node_metadata(self): - dataset = PartitionedDataset(path="partitioned/", dataset="pandas.CSVDataSet") + dataset = PartitionedDataset(path="partitioned/", dataset="pandas.CSVDataset") data_node = GraphNode.create_data_node( dataset_name="dataset", layer="raw", From f00e75cae67902807c98d7353f562f0a89eedf84 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Wed, 11 Oct 2023 15:17:37 +0100 Subject: [PATCH 2/5] Fix lint + unpin docer reqs Signed-off-by: Merel Theisen --- demo-project/src/docker_requirements.txt | 2 +- package/kedro_viz/models/flowchart.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/demo-project/src/docker_requirements.txt b/demo-project/src/docker_requirements.txt index 515c0c0b3..26910d094 100644 --- a/demo-project/src/docker_requirements.txt +++ b/demo-project/src/docker_requirements.txt @@ -1,5 +1,5 @@ kedro~=0.18.0 -kedro-datasets[pandas.CSVDataSet,pandas.ExcelDataSet, pandas.ParquetDataSet, plotly.PlotlyDataSet, matplotlib.MatplotlibWriter]>=1.0, <1.7.1 +kedro-datasets[pandas.CSVDataSet,pandas.ExcelDataSet, pandas.ParquetDataSet, plotly.PlotlyDataSet, matplotlib.MatplotlibWriter]~=1.0 scikit-learn~=1.0 pillow~=9.0 seaborn~=0.11.2 diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 0a0be5e02..fb32902e6 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -478,13 +478,15 @@ def is_metric_node(self): """Check if the current node is a metrics node.""" return self.dataset_type in ( "tracking.metrics_dataset.MetricsDataSet", - "tracking.metrics_dataset.MetricsDataset") + "tracking.metrics_dataset.MetricsDataset", + ) def is_json_node(self): """Check if the current node is a JSONDataset node.""" return self.dataset_type in ( "tracking.json_dataset.JSONDataSet", - "tracking.json_dataset.JSONDataset") + "tracking.json_dataset.JSONDataset", + ) def is_tracking_node(self): """Checks if the current node is a tracking data node""" From 7542612cd81ab2abca751a05474f0d528f8fd449 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 12 Oct 2023 12:46:15 +0100 Subject: [PATCH 3/5] Make sure experiment tracking works with new dataset names Signed-off-by: Merel Theisen --- src/config.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config.js b/src/config.js index 8ead53246..08872481b 100644 --- a/src/config.js +++ b/src/config.js @@ -98,10 +98,14 @@ export const sidebarElementTypes = { export const shortTypeMapping = { 'plotly.plotly_dataset.PlotlyDataSet': 'plotly', + 'plotly.plotly_dataset.PlotlyDataset': 'plotly', 'plotly.json_dataset.JSONDataSet': 'plotly', + 'plotly.json_dataset.JSONDataset': 'plotly', 'matplotlib.matplotlib_writer.MatplotlibWriter': 'image', 'tracking.json_dataset.JSONDataSet': 'JSONTracking', + 'tracking.json_dataset.JSONDataset': 'JSONTracking', 'tracking.metrics_dataset.MetricsDataSet': 'metricsTracking', + 'tracking.metrics_dataset.MetricsDataset': 'metricsTracking', }; export const tabLabels = ['Overview', 'Metrics', 'Plots']; From f91b89391bc8dc552eeb7d80e2deaba9eec2f839 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 12 Oct 2023 13:51:33 +0100 Subject: [PATCH 4/5] Try rename all mentions of DataSet to Dataset Signed-off-by: Merel Theisen --- README.md | 2 +- .../fixtures/graphql/compareThreeRuns.json | 12 +-- cypress/fixtures/graphql/compareTwoRuns.json | 12 +-- cypress/fixtures/graphql/getRunData.json | 10 +- cypress/fixtures/mock/largeDataset.json | 40 ++++---- package-lock.json | 32 ++++++- .../integrations/kedro/data_loader.py | 24 ----- .../kedro_viz/models/experiment_tracking.py | 4 - package/kedro_viz/models/flowchart.py | 4 - .../run-dataset/run-dataset-loader.js | 2 +- .../run-dataset/run-dataset.js | 6 +- .../run-dataset/run-dataset.test.js | 12 +-- .../run-plots-modal/run-plots-modal.test.js | 2 +- src/components/metadata/metadata.test.js | 40 ++++---- src/config.js | 4 - src/utils/data/demo.mock.json | 92 +++++++++---------- src/utils/data/node_data.mock.json | 2 +- src/utils/data/node_data_stats.mock.json | 2 +- src/utils/data/node_json_data.mock.json | 2 +- src/utils/data/node_metrics_data.mock.json | 2 +- src/utils/data/node_plot.mock.json | 2 +- src/utils/data/node_transcoded_data.mock.json | 4 +- src/utils/data/spaceflights.mock.json | 16 ++-- 23 files changed, 160 insertions(+), 168 deletions(-) diff --git a/README.md b/README.md index 3d5db6aa2..3567ffce3 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ Once the above set-up is complete, tracking datasets can be used to track releva ```yaml train_evaluation.r2_score_linear_regression: - type: tracking.MetricsDataSet + type: tracking.MetricsDataset filepath: ${base_location}/09_tracking/linear_score.json versioned: true ``` diff --git a/cypress/fixtures/graphql/compareThreeRuns.json b/cypress/fixtures/graphql/compareThreeRuns.json index 0702a259b..aa163beaf 100644 --- a/cypress/fixtures/graphql/compareThreeRuns.json +++ b/cypress/fixtures/graphql/compareThreeRuns.json @@ -2968,7 +2968,7 @@ ] }, "datasetName": "reporting.feature_importance", - "datasetType": "plotly.json_dataset.JSONDataSet", + "datasetType": "plotly.json_dataset.JSONDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z", @@ -3050,7 +3050,7 @@ ] }, "datasetName": "train_evaluation.linear_regression.r2_score", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z", @@ -3104,7 +3104,7 @@ ] }, "datasetName": "train_evaluation.random_forest.r2_score", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z", @@ -3199,7 +3199,7 @@ ] }, "datasetName": "train_evaluation.linear_regression.experiment_params", - "datasetType": "tracking.json_dataset.JSONDataSet", + "datasetType": "tracking.json_dataset.JSONDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z", @@ -3393,7 +3393,7 @@ ] }, "datasetName": "train_evaluation.random_forest.experiment_params", - "datasetType": "tracking.json_dataset.JSONDataSet", + "datasetType": "tracking.json_dataset.JSONDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z", @@ -3403,4 +3403,4 @@ } ] } -} \ No newline at end of file +} diff --git a/cypress/fixtures/graphql/compareTwoRuns.json b/cypress/fixtures/graphql/compareTwoRuns.json index 6f3a92d35..56198ca42 100644 --- a/cypress/fixtures/graphql/compareTwoRuns.json +++ b/cypress/fixtures/graphql/compareTwoRuns.json @@ -1982,7 +1982,7 @@ ] }, "datasetName": "reporting.feature_importance", - "datasetType": "plotly.json_dataset.JSONDataSet", + "datasetType": "plotly.json_dataset.JSONDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z" @@ -2046,7 +2046,7 @@ ] }, "datasetName": "train_evaluation.linear_regression.r2_score", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z" @@ -2087,7 +2087,7 @@ ] }, "datasetName": "train_evaluation.random_forest.r2_score", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z" @@ -2165,7 +2165,7 @@ ] }, "datasetName": "train_evaluation.linear_regression.experiment_params", - "datasetType": "tracking.json_dataset.JSONDataSet", + "datasetType": "tracking.json_dataset.JSONDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z" @@ -2306,7 +2306,7 @@ ] }, "datasetName": "train_evaluation.random_forest.experiment_params", - "datasetType": "tracking.json_dataset.JSONDataSet", + "datasetType": "tracking.json_dataset.JSONDataset", "runIds": [ "2022-12-24T21.05.59.296Z", "2022-10-05T12.22.35.825Z" @@ -2315,4 +2315,4 @@ } ] } -} \ No newline at end of file +} diff --git a/cypress/fixtures/graphql/getRunData.json b/cypress/fixtures/graphql/getRunData.json index 5ec27c085..04c89bedb 100644 --- a/cypress/fixtures/graphql/getRunData.json +++ b/cypress/fixtures/graphql/getRunData.json @@ -668,7 +668,7 @@ ] }, "datasetName": "reporting.feature_importance", - "datasetType": "plotly.json_dataset.JSONDataSet", + "datasetType": "plotly.json_dataset.JSONDataset", "runIds": ["2022-12-24T21.05.59.296Z"], "__typename": "TrackingDataset" }, @@ -710,7 +710,7 @@ ] }, "datasetName": "train_evaluation.linear_regression.r2_score", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "runIds": ["2022-12-24T21.05.59.296Z"], "__typename": "TrackingDataset" }, @@ -736,7 +736,7 @@ ] }, "datasetName": "train_evaluation.random_forest.r2_score", - "datasetType": "tracking.metrics_dataset.MetricsDataSet", + "datasetType": "tracking.metrics_dataset.MetricsDataset", "runIds": ["2022-12-24T21.05.59.296Z"], "__typename": "TrackingDataset" } @@ -786,7 +786,7 @@ ] }, "datasetName": "train_evaluation.linear_regression.experiment_params", - "datasetType": "tracking.json_dataset.JSONDataSet", + "datasetType": "tracking.json_dataset.JSONDataset", "runIds": ["2022-12-24T21.05.59.296Z"], "__typename": "TrackingDataset" }, @@ -872,7 +872,7 @@ ] }, "datasetName": "train_evaluation.random_forest.experiment_params", - "datasetType": "tracking.json_dataset.JSONDataSet", + "datasetType": "tracking.json_dataset.JSONDataset", "runIds": ["2022-12-24T21.05.59.296Z"], "__typename": "TrackingDataset" } diff --git a/cypress/fixtures/mock/largeDataset.json b/cypress/fixtures/mock/largeDataset.json index 1dd7a30d8..5499917a1 100644 --- a/cypress/fixtures/mock/largeDataset.json +++ b/cypress/fixtures/mock/largeDataset.json @@ -48,7 +48,7 @@ "type": "data", "modular_pipelines": [], "layer": "primary", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "9f266f06", @@ -63,7 +63,7 @@ "type": "data", "modular_pipelines": [], "layer": "primary", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "abed6a4d", @@ -126,7 +126,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "9c43f772", @@ -215,7 +215,7 @@ "type": "data", "modular_pipelines": [], "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "f23ad217", @@ -231,7 +231,7 @@ "ingestion" ], "layer": "intermediate", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "82d36a1b", @@ -264,7 +264,7 @@ "type": "data", "modular_pipelines": [], "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "b5609df0", @@ -298,7 +298,7 @@ "ingestion" ], "layer": "intermediate", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "f33b9291", @@ -327,7 +327,7 @@ "type": "data", "modular_pipelines": [], "layer": "raw", - "dataset_type": "pandas.excel_dataset.ExcelDataSet" + "dataset_type": "pandas.excel_dataset.ExcelDataset" }, { "id": "c0ddbcbf", @@ -401,7 +401,7 @@ "reporting" ], "layer": "reporting", - "dataset_type": "plotly.plotly_dataset.PlotlyDataSet" + "dataset_type": "plotly.plotly_dataset.PlotlyDataset" }, { "id": "3fb71518", @@ -430,7 +430,7 @@ "reporting" ], "layer": null, - "dataset_type": "datasets.image_dataset.ImageDataSet" + "dataset_type": "datasets.image_dataset.ImageDataset" }, { "id": "40886786", @@ -459,7 +459,7 @@ "reporting" ], "layer": "reporting", - "dataset_type": "plotly.json_dataset.JSONDataSet" + "dataset_type": "plotly.json_dataset.JSONDataset" }, { "id": "178d37bb", @@ -489,7 +489,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_input", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "8de402c1", @@ -549,7 +549,7 @@ "reporting" ], "layer": "reporting", - "dataset_type": "plotly.json_dataset.JSONDataSet" + "dataset_type": "plotly.json_dataset.JSONDataset" }, { "id": "25b1f0cf", @@ -29736,7 +29736,7 @@ "train_evaluation.linear_regression" ], "layer": null, - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "b701864d", @@ -29752,7 +29752,7 @@ "train_evaluation.linear_regression" ], "layer": null, - "dataset_type": "tracking.json_dataset.JSONDataSet" + "dataset_type": "tracking.json_dataset.JSONDataset" }, { "id": "1c0614b4", @@ -29818,7 +29818,7 @@ "train_evaluation.random_forest" ], "layer": null, - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "4f79de77", @@ -29834,7 +29834,7 @@ "train_evaluation.random_forest" ], "layer": null, - "dataset_type": "tracking.json_dataset.JSONDataSet" + "dataset_type": "tracking.json_dataset.JSONDataset" }, { "id": "d6a09df8", @@ -29865,7 +29865,7 @@ "train_evaluation.linear_regression" ], "layer": null, - "dataset_type": "tracking.metrics_dataset.MetricsDataSet" + "dataset_type": "tracking.metrics_dataset.MetricsDataset" }, { "id": "0b70ae9d", @@ -29896,7 +29896,7 @@ "train_evaluation.random_forest" ], "layer": null, - "dataset_type": "tracking.metrics_dataset.MetricsDataSet" + "dataset_type": "tracking.metrics_dataset.MetricsDataset" }, { "id": "feature_engineering", @@ -51765,4 +51765,4 @@ } }, "selected_pipeline": "__default__" - } \ No newline at end of file + } diff --git a/package-lock.json b/package-lock.json index 32bfaef27..49f618f7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@quantumblack/kedro-viz", - "version": "6.5.0", + "version": "6.6.0", "dependencies": { "@apollo/client": "^3.5.6", "@emotion/react": "^11.10.6", @@ -68,6 +68,7 @@ "@wojtekmaj/enzyme-adapter-react-17": "^0.8.0", "babel-plugin-transform-remove-imports": "^1.7.0", "canvas": "^2.7.0", + "cross-env": "7.0.3", "css-loader": "^4.3.0", "cypress": "^12.14.0", "enzyme": "^3.11.0", @@ -9985,6 +9986,24 @@ "sha.js": "^2.4.8" } }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-fetch": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", @@ -45429,6 +45448,15 @@ "sha.js": "^2.4.8" } }, + "cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.1" + } + }, "cross-fetch": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", @@ -67556,4 +67584,4 @@ } } } -} \ No newline at end of file +} diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index 812d720b8..1542ecf1e 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -175,48 +175,24 @@ def matplotlib_writer_load(dataset: matplotlib.MatplotlibWriter) -> str: except (ImportError, AttributeError): pass -try: - getattr(plotly, "JSONDataSet") # Trigger import - plotly.JSONDataSet._load = json_dataset.JSONDataSet._load -except (ImportError, AttributeError): - pass - try: getattr(plotly, "JSONDataset") # Trigger import plotly.JSONDataset._load = json_dataset.JSONDataset._load except (ImportError, AttributeError): pass -try: - getattr(plotly, "PlotlyDataSet") # Trigger import - plotly.PlotlyDataSet._load = json_dataset.JSONDataSet._load -except (ImportError, AttributeError): - pass - try: getattr(plotly, "PlotlyDataset") # Trigger import plotly.PlotlyDataset._load = json_dataset.JSONDataset._load except (ImportError, AttributeError): pass -try: - getattr(tracking, "JSONDataSet") # Trigger import - tracking.JSONDataSet._load = json_dataset.JSONDataSet._load -except (ImportError, AttributeError): - pass - try: getattr(tracking, "JSONDataset") # Trigger import tracking.JSONDataset._load = json_dataset.JSONDataset._load except (ImportError, AttributeError): pass -try: - getattr(tracking, "MetricsDataSet") # Trigger import - tracking.MetricsDataSet._load = json_dataset.JSONDataSet._load -except (ImportError, AttributeError): - pass - try: getattr(tracking, "MetricsDataset") # Trigger import tracking.MetricsDataset._load = json_dataset.JSONDataset._load diff --git a/package/kedro_viz/models/experiment_tracking.py b/package/kedro_viz/models/experiment_tracking.py index 63debe17e..ef260801f 100644 --- a/package/kedro_viz/models/experiment_tracking.py +++ b/package/kedro_viz/models/experiment_tracking.py @@ -63,14 +63,10 @@ class TrackingDatasetGroup(str, Enum): # Map dataset types (as produced by get_dataset_type) to their group TRACKING_DATASET_GROUPS = { - "plotly.plotly_dataset.PlotlyDataSet": TrackingDatasetGroup.PLOT, "plotly.plotly_dataset.PlotlyDataset": TrackingDatasetGroup.PLOT, - "plotly.json_dataset.JSONDataSet": TrackingDatasetGroup.PLOT, "plotly.json_dataset.JSONDataset": TrackingDatasetGroup.PLOT, "matplotlib.matplotlib_writer.MatplotlibWriter": TrackingDatasetGroup.PLOT, - "tracking.metrics_dataset.MetricsDataSet": TrackingDatasetGroup.METRIC, "tracking.metrics_dataset.MetricsDataset": TrackingDatasetGroup.METRIC, - "tracking.json_dataset.JSONDataSet": TrackingDatasetGroup.JSON, "tracking.json_dataset.JSONDataset": TrackingDatasetGroup.JSON, } diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index fb32902e6..bd6a20953 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -464,9 +464,7 @@ def is_plot_node(self): In the future, we might want to make this generic. """ return self.dataset_type in ( - "plotly.plotly_dataset.PlotlyDataSet", "plotly.plotly_dataset.PlotlyDataset", - "plotly.json_dataset.JSONDataSet", "plotly.json_dataset.JSONDataset", ) @@ -477,14 +475,12 @@ def is_image_node(self): def is_metric_node(self): """Check if the current node is a metrics node.""" return self.dataset_type in ( - "tracking.metrics_dataset.MetricsDataSet", "tracking.metrics_dataset.MetricsDataset", ) def is_json_node(self): """Check if the current node is a JSONDataset node.""" return self.dataset_type in ( - "tracking.json_dataset.JSONDataSet", "tracking.json_dataset.JSONDataset", ) diff --git a/src/components/experiment-tracking/run-dataset/run-dataset-loader.js b/src/components/experiment-tracking/run-dataset/run-dataset-loader.js index f2db4fd93..bd0cd4d23 100644 --- a/src/components/experiment-tracking/run-dataset/run-dataset-loader.js +++ b/src/components/experiment-tracking/run-dataset/run-dataset-loader.js @@ -66,7 +66,7 @@ export const SingleRunDatasetLoader = ({ theme }) => ( ); -export const DataSetLoader = ({ x, y, length, theme }) => { +export const DatasetLoader = ({ x, y, length, theme }) => { return ( {showLoader && ( - {showLoader && ( - { if (dataType === 'matplotlib') { return { diff --git a/src/components/metadata/metadata.test.js b/src/components/metadata/metadata.test.js index 7fcfc9c57..12dc5b584 100644 --- a/src/components/metadata/metadata.test.js +++ b/src/components/metadata/metadata.test.js @@ -15,7 +15,7 @@ import nodeMetricsData from '../../utils/data/node_metrics_data.mock.json'; import nodeJSONData from '../../utils/data/node_json_data.mock.json'; import { formatFileSize } from '../../utils'; -const modelInputDataSetNodeId = '23c94afb'; +const modelInputDatasetNodeId = '23c94afb'; const splitDataTaskNodeId = '65d0d789'; const parametersNodeId = 'f1f1425b'; const dataScienceNodeId = 'data_science'; @@ -272,7 +272,7 @@ describe('MetaData', () => { describe('Dataset nodes', () => { it('shows the node type as an icon', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeData, }); expect(rowIcon(wrapper).hasClass('pipeline-node-icon--icon-data')).toBe( @@ -282,7 +282,7 @@ describe('MetaData', () => { it('shows the node name as the title', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeData, }); expect(textOf(title(wrapper))).toEqual(['Model Input Table']); @@ -290,7 +290,7 @@ describe('MetaData', () => { it('shows the node type as text', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeData, }); const row = rowByLabel(wrapper, 'Type:'); @@ -299,16 +299,16 @@ describe('MetaData', () => { it('shows the node dataset type', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeData, }); const row = rowByLabel(wrapper, 'Dataset Type:'); - expect(textOf(rowValue(row))).toEqual(['CSVDataSet']); + expect(textOf(rowValue(row))).toEqual(['CSVDataset']); }); it('shows the node filepath', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeData, }); const row = rowByLabel(wrapper, 'File Path:'); @@ -319,7 +319,7 @@ describe('MetaData', () => { it('wont show any tags as they should only appear if the type is nodeTask', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeData, }); const row = rowByLabel(wrapper, 'Tags:'); @@ -329,7 +329,7 @@ describe('MetaData', () => { describe('when there is a runCommand returned by the backend', () => { it('shows the node run command', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeData, }); @@ -345,7 +345,7 @@ describe('MetaData', () => { }; const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeData, }); @@ -362,7 +362,7 @@ describe('MetaData', () => { describe('when there is stats returned by the backend', () => { it('shows the node statistics', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeDataStats, }); @@ -392,26 +392,26 @@ describe('MetaData', () => { describe('Transcoded dataset nodes', () => { it('shows the node original type', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeTranscodedData, }); const row = rowByLabel(wrapper, 'Original Type:'); - expect(textOf(rowValue(row))).toEqual(['SparkDataSet']); + expect(textOf(rowValue(row))).toEqual(['SparkDataset']); }); it('shows the node transcoded type', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeTranscodedData, }); const row = rowByLabel(wrapper, 'Transcoded Types:'); - expect(textOf(rowValue(row))).toEqual(['ParquetDataSet']); + expect(textOf(rowValue(row))).toEqual(['ParquetDataset']); }); }); describe('Metrics dataset nodes', () => { it('shows the node metrics', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeMetricsData, }); const row = rowByLabel(wrapper, 'Tracking data from last run:'); @@ -421,7 +421,7 @@ describe('MetaData', () => { }); it('shows the experiment link', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeMetricsData, }); expect(wrapper.find('.pipeline-metadata__link').length).toBe(1); @@ -431,7 +431,7 @@ describe('MetaData', () => { describe('JSON dataset nodes', () => { it('shows the json data', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeJSONData, }); const row = rowByLabel(wrapper, 'Tracking data from last run:'); @@ -441,7 +441,7 @@ describe('MetaData', () => { }); it('shows the experiment link', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodeJSONData, }); expect(wrapper.find('.pipeline-metadata__link').length).toBe(1); @@ -451,7 +451,7 @@ describe('MetaData', () => { describe('Plot nodes', () => { describe('shows the plot info', () => { const wrapper = mount({ - nodeId: modelInputDataSetNodeId, + nodeId: modelInputDatasetNodeId, mockMetadata: nodePlot, }); it('shows the plotly chart', () => { diff --git a/src/config.js b/src/config.js index 08872481b..d269f71f5 100644 --- a/src/config.js +++ b/src/config.js @@ -97,14 +97,10 @@ export const sidebarElementTypes = { }; export const shortTypeMapping = { - 'plotly.plotly_dataset.PlotlyDataSet': 'plotly', 'plotly.plotly_dataset.PlotlyDataset': 'plotly', - 'plotly.json_dataset.JSONDataSet': 'plotly', 'plotly.json_dataset.JSONDataset': 'plotly', 'matplotlib.matplotlib_writer.MatplotlibWriter': 'image', - 'tracking.json_dataset.JSONDataSet': 'JSONTracking', 'tracking.json_dataset.JSONDataset': 'JSONTracking', - 'tracking.metrics_dataset.MetricsDataSet': 'metricsTracking', 'tracking.metrics_dataset.MetricsDataset': 'metricsTracking', }; diff --git a/src/utils/data/demo.mock.json b/src/utils/data/demo.mock.json index 0b4171359..470bba714 100644 --- a/src/utils/data/demo.mock.json +++ b/src/utils/data/demo.mock.json @@ -33,7 +33,7 @@ "type": "data", "modular_pipelines": [], "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "7a65da0d", @@ -75,7 +75,7 @@ "type": "data", "modular_pipelines": [], "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "95cd6bf9", @@ -111,7 +111,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "845526ea", @@ -146,7 +146,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "16cae681", @@ -185,7 +185,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "ddc6c97b", @@ -222,7 +222,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "3464b488", @@ -312,7 +312,7 @@ "type": "data", "modular_pipelines": [], "layer": "intermediate", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "d28f4db6", @@ -322,7 +322,7 @@ "type": "data", "modular_pipelines": [], "layer": "intermediate", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "31fb7cc8", @@ -358,7 +358,7 @@ "type": "data", "modular_pipelines": [], "layer": "intermediate", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "a771381b", @@ -368,7 +368,7 @@ "type": "data", "modular_pipelines": [], "layer": "intermediate", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "dec9c820", @@ -1017,7 +1017,7 @@ "type": "data", "modular_pipelines": [], "layer": "raw", - "dataset_type": "shuttle_factory.utilities.io.tag_dict_datasets.CustomCSVDataSet" + "dataset_type": "shuttle_factory.utilities.io.tag_dict_datasets.CustomCSVDataset" }, { "id": "c3f72bfd", @@ -1116,7 +1116,7 @@ "type": "data", "modular_pipelines": [], "layer": "primary", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "893a3f7f", @@ -1148,7 +1148,7 @@ "type": "data", "modular_pipelines": [], "layer": "primary", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "b10165f4", @@ -1180,7 +1180,7 @@ "type": "data", "modular_pipelines": [], "layer": "primary", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "93d3a479", @@ -1212,7 +1212,7 @@ "type": "data", "modular_pipelines": [], "layer": "primary", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "7be9f72e", @@ -1243,7 +1243,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "11a72273", @@ -1274,7 +1274,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "22cfe9cd", @@ -1293,7 +1293,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "2978a7f5", @@ -1312,7 +1312,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "72ac6e43", @@ -1331,7 +1331,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "1d7fffba", @@ -1350,7 +1350,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "fe81cbe2", @@ -1387,7 +1387,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "690d415b", @@ -1414,7 +1414,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "eb1aa767", @@ -1433,7 +1433,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "4b7d240f", @@ -1452,7 +1452,7 @@ "type": "data", "modular_pipelines": [], "layer": "feature", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "a8bef9b5", @@ -1696,7 +1696,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_input", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "2ea5719f", @@ -1727,7 +1727,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_input", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "602f06fc", @@ -2038,7 +2038,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_output", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "10ce7f66", @@ -2048,7 +2048,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_output", - "dataset_type": "pandas.json_dataset.JSONDataSet" + "dataset_type": "pandas.json_dataset.JSONDataset" }, { "id": "e6c1e1d0", @@ -2058,7 +2058,7 @@ "type": "data", "modular_pipelines": [], "layer": "models", - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "68d6f757", @@ -2249,7 +2249,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "9a97cfe0", @@ -2259,7 +2259,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_output", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "c0020a3e", @@ -2349,7 +2349,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "9b9921db", @@ -2359,7 +2359,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "c329efd1", @@ -2401,7 +2401,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "ae98f5f5", @@ -2450,7 +2450,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "tracking.metrics_dataset.MetricsDataSet" + "dataset_type": "tracking.metrics_dataset.MetricsDataset" }, { "id": "d1733133", @@ -2473,7 +2473,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_output", - "dataset_type": "pandas.json_dataset.JSONDataSet" + "dataset_type": "pandas.json_dataset.JSONDataset" }, { "id": "b7c5a4e0", @@ -2483,7 +2483,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_output", - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "24c3ce33", @@ -2493,7 +2493,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_output", - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "7ba4e963", @@ -2503,7 +2503,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_output", - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "70c65095", @@ -2513,7 +2513,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_output", - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "55758695", @@ -2532,7 +2532,7 @@ "type": "data", "modular_pipelines": [], "layer": "model_output", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "bf435119", @@ -2570,7 +2570,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "plotly.plotly_dataset.PlotlyDataSet" + "dataset_type": "plotly.plotly_dataset.PlotlyDataset" }, { "id": "0298a817", @@ -2613,7 +2613,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "f8e6b6f3", @@ -2632,7 +2632,7 @@ "type": "data", "modular_pipelines": [], "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "6526dd55", @@ -2661,7 +2661,7 @@ "type": "data", "modular_pipelines": [], "layer": "primary", - "dataset_type": "pandas.parquet_dataset.ParquetDataSet" + "dataset_type": "pandas.parquet_dataset.ParquetDataset" }, { "id": "factory.pandas_profiling", diff --git a/src/utils/data/node_data.mock.json b/src/utils/data/node_data.mock.json index fef18d408..bc27ac136 100644 --- a/src/utils/data/node_data.mock.json +++ b/src/utils/data/node_data.mock.json @@ -1,5 +1,5 @@ { "filepath": "/tmp/project/data/03_primary/model_input_table.csv", - "type": "pandas.csv_dataset.CSVDataSet", + "type": "pandas.csv_dataset.CSVDataset", "run_command": "kedro run --to-outputs=model_input_table" } diff --git a/src/utils/data/node_data_stats.mock.json b/src/utils/data/node_data_stats.mock.json index 7e6c07bca..84e851940 100644 --- a/src/utils/data/node_data_stats.mock.json +++ b/src/utils/data/node_data_stats.mock.json @@ -1,6 +1,6 @@ { "filepath": "/tmp/project/data/03_primary/model_input_table.csv", - "type": "pandas.csv_dataset.CSVDataSet", + "type": "pandas.csv_dataset.CSVDataset", "run_command": "kedro run --to-outputs=model_input_table", "stats": { "rows": 10, diff --git a/src/utils/data/node_json_data.mock.json b/src/utils/data/node_json_data.mock.json index ceed86f64..42e0f4ec5 100644 --- a/src/utils/data/node_json_data.mock.json +++ b/src/utils/data/node_json_data.mock.json @@ -1,6 +1,6 @@ { "filepath": "/Users/Documents/project-src/test/data/01_raw/iris.csv", - "type": "tracking.json_dataset.JSONDataSet", + "type": "tracking.json_dataset.JSONDataset", "tracking_data": { "recommendations": "dummy_recommendation", "recommended_controls": 0.2701227292578884, diff --git a/src/utils/data/node_metrics_data.mock.json b/src/utils/data/node_metrics_data.mock.json index 50ed6f0f7..575fc8c4f 100644 --- a/src/utils/data/node_metrics_data.mock.json +++ b/src/utils/data/node_metrics_data.mock.json @@ -1,6 +1,6 @@ { "filepath": "/Users/Documents/project-src/test/data/01_raw/iris.csv", - "type": "tracking.metrics_dataset.MetricsDataSet", + "type": "tracking.metrics_dataset.MetricsDataset", "tracking_data": { "recommendations": 0.2160981834063107, "recommended_controls": 0.2701227292578884, diff --git a/src/utils/data/node_plot.mock.json b/src/utils/data/node_plot.mock.json index c18f46a82..b9388935e 100644 --- a/src/utils/data/node_plot.mock.json +++ b/src/utils/data/node_plot.mock.json @@ -1,5 +1,5 @@ { - "type": "plotly.plotly_dataset.PlotlyDataSet", + "type": "plotly.plotly_dataset.PlotlyDataset", "plot": { "data": [ { diff --git a/src/utils/data/node_transcoded_data.mock.json b/src/utils/data/node_transcoded_data.mock.json index 9590da1bd..cbcc22555 100644 --- a/src/utils/data/node_transcoded_data.mock.json +++ b/src/utils/data/node_transcoded_data.mock.json @@ -1,5 +1,5 @@ { "filepath": "/Users/Documents/project-src/test/data/01_raw/iris.csv", - "original_type": "spark.spark_dataset.SparkDataSet", - "transcoded_types": ["pandas.parquet_dataset.ParquetDataSet"] + "original_type": "spark.spark_dataset.SparkDataset", + "transcoded_types": ["pandas.parquet_dataset.ParquetDataset"] } diff --git a/src/utils/data/spaceflights.mock.json b/src/utils/data/spaceflights.mock.json index ec4722433..86f01565d 100644 --- a/src/utils/data/spaceflights.mock.json +++ b/src/utils/data/spaceflights.mock.json @@ -17,7 +17,7 @@ "type": "data", "modular_pipelines": ["data_processing"], "layer": "raw", - "dataset_type": "pandas.excel_dataset.ExcelDataSet" + "dataset_type": "pandas.excel_dataset.ExcelDataset" }, { "id": "e5a9ec27", @@ -27,7 +27,7 @@ "type": "data", "modular_pipelines": ["data_processing"], "layer": "intermediate", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "c09084f2", @@ -46,7 +46,7 @@ "type": "data", "modular_pipelines": ["data_processing"], "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "daf35ba0", @@ -56,7 +56,7 @@ "type": "data", "modular_pipelines": ["data_processing"], "layer": "intermediate", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "47b81aa6", @@ -75,7 +75,7 @@ "type": "data", "modular_pipelines": ["data_processing"], "layer": "raw", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "23c94afb", @@ -85,7 +85,7 @@ "type": "data", "modular_pipelines": [], "layer": "primary", - "dataset_type": "pandas.csv_dataset.CSVDataSet" + "dataset_type": "pandas.csv_dataset.CSVDataset" }, { "id": "65d0d789", @@ -176,7 +176,7 @@ "type": "data", "modular_pipelines": ["data_science"], "layer": "models", - "dataset_type": "pickle.pickle_dataset.PickleDataSet" + "dataset_type": "pickle.pickle_dataset.PickleDataset" }, { "id": "f5e8d7df", @@ -195,7 +195,7 @@ "type": "data", "modular_pipelines": [], "layer": null, - "dataset_type": "tracking.metrics_dataset.MetricsDataSet" + "dataset_type": "tracking.metrics_dataset.MetricsDataset" }, { "id": "data_processing", From 96cf33e1da8118d647b4a49077065f356d7f4e9c Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 12 Oct 2023 14:07:17 +0100 Subject: [PATCH 5/5] Fix lint Signed-off-by: Merel Theisen --- package/kedro_viz/models/flowchart.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index bd6a20953..05f0dd364 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -474,15 +474,11 @@ def is_image_node(self): def is_metric_node(self): """Check if the current node is a metrics node.""" - return self.dataset_type in ( - "tracking.metrics_dataset.MetricsDataset", - ) + return self.dataset_type in ("tracking.metrics_dataset.MetricsDataset",) def is_json_node(self): """Check if the current node is a JSONDataset node.""" - return self.dataset_type in ( - "tracking.json_dataset.JSONDataset", - ) + return self.dataset_type in ("tracking.json_dataset.JSONDataset",) def is_tracking_node(self): """Checks if the current node is a tracking data node"""