chore: Drop support for python 3.8 on kedro-datasets (#442)

* Drop support for python 3.8 on kedro-datasets

---------

Signed-off-by: Dmitry Sorokin <[email protected]>
Signed-off-by: Dmitry Sorokin <[email protected]>
Co-authored-by: Merel Theisen <[email protected]>

.github/workflows/kedro-datasets.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, windows-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.9", "3.10", "3.11" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-datasets
@@ -41,15 +41,15 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v3
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Cache python packages
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: kedro-datasets-ubuntu-latest-python-"3.8"
+          key: kedro-datasets-ubuntu-latest-python-"3.9"
           restore-keys: kedro-datasets
       - name: Install dependencies
         run: |

kedro-datasets/.readthedocs.yaml

@@ -8,7 +8,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.8"
+    python: "3.9"
   jobs:
     pre_install:
     # pip==23.2 breaks pip-tools<7.0, and pip-tools>=7.0 does not support Python 3.7

kedro-datasets/README.md

@@ -1,7 +1,7 @@
 # Kedro-Datasets
 
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
-[![Python Version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11-blue.svg)](https://pypi.org/project/kedro-datasets/)
+[![Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11-blue.svg)](https://pypi.org/project/kedro-datasets/)
 [![PyPI Version](https://badge.fury.io/py/kedro-datasets.svg)](https://pypi.org/project/kedro-datasets/)
 [![Code Style: Black](https://img.shields.io/badge/code%20style-black-black.svg)](https://github.com/ambv/black)
 

kedro-datasets/RELEASE.md

@@ -1,7 +1,7 @@
 # Upcoming Release
 
 ## Major features and improvements
-* Removed support for Python 3.7
+* Removed support for Python 3.7 and 3.8
 * Spark and Databricks based datasets now support [databricks-connect>=13.0](https://docs.databricks.com/en/dev-tools/databricks-connect-ref.html)
 
 ## Bug fixes and other changes

kedro-datasets/docs/source/conf.py

@@ -99,6 +99,7 @@
     "py:class": (
         "kedro.io.core.AbstractDataset",
         "kedro.io.AbstractDataset",
+        "AbstractDataset",
         "kedro.io.core.Version",
         "requests.auth.AuthBase",
         "google.oauth2.credentials.Credentials",

kedro-datasets/kedro_datasets/api/api_dataset.py

@@ -3,7 +3,7 @@
 """
 import json as json_  # make pylint happy
 from copy import deepcopy
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Union
 
 import requests
 from requests import Session, sessions
@@ -93,10 +93,10 @@ def __init__(  # noqa: PLR0913
         *,
         url: str,
         method: str = "GET",
-        load_args: Dict[str, Any] = None,
-        save_args: Dict[str, Any] = None,
-        credentials: Union[Tuple[str, str], List[str], AuthBase] = None,
-        metadata: Dict[str, Any] = None,
+        load_args: dict[str, Any] = None,
+        save_args: dict[str, Any] = None,
+        credentials: Union[tuple[str, str], list[str], AuthBase] = None,
+        metadata: dict[str, Any] = None,
     ) -> None:
         """Creates a new instance of ``APIDataset`` to fetch data from an API endpoint.
 
@@ -147,7 +147,7 @@ def __init__(  # noqa: PLR0913
         if "timeout" in self._params:
             self._params["timeout"] = self._convert_type(self._params["timeout"])
 
-        self._request_args: Dict[str, Any] = {
+        self._request_args: dict[str, Any] = {
             "url": url,
             "method": method,
             "auth": self._convert_type(self._auth),
@@ -163,11 +163,11 @@ def _convert_type(value: Any):
         However, for some parameters in the Python requests library,
         only Tuples are allowed.
         """
-        if isinstance(value, List):
+        if isinstance(value, list):
             return tuple(value)
         return value
 
-    def _describe(self) -> Dict[str, Any]:
+    def _describe(self) -> dict[str, Any]:
         # prevent auth from logging
         request_args_cp = self._request_args.copy()
         request_args_cp.pop("auth", None)
@@ -193,7 +193,7 @@ def _load(self) -> requests.Response:
 
     def _execute_save_with_chunks(
         self,
-        json_data: List[Dict[str, Any]],
+        json_data: list[dict[str, Any]],
     ) -> requests.Response:
         chunk_size = self._chunk_size
         n_chunks = len(json_data) // chunk_size + 1

kedro-datasets/kedro_datasets/biosequence/biosequence_dataset.py

@@ -3,7 +3,7 @@
 """
 from copy import deepcopy
 from pathlib import PurePosixPath
-from typing import Any, Dict, List
+from typing import Any
 
 import fsspec
 from Bio import SeqIO
@@ -12,7 +12,7 @@
 from kedro_datasets._io import AbstractDataset
 
 
-class BioSequenceDataset(AbstractDataset[List, List]):
+class BioSequenceDataset(AbstractDataset[list, list]):
     r"""``BioSequenceDataset`` loads and saves data to a sequence file.
 
     Example:
@@ -42,18 +42,18 @@ class BioSequenceDataset(AbstractDataset[List, List]):
 
     """
 
-    DEFAULT_LOAD_ARGS: Dict[str, Any] = {}
-    DEFAULT_SAVE_ARGS: Dict[str, Any] = {}
+    DEFAULT_LOAD_ARGS: dict[str, Any] = {}
+    DEFAULT_SAVE_ARGS: dict[str, Any] = {}
 
     def __init__(  # noqa: PLR0913
         self,
         *,
         filepath: str,
-        load_args: Dict[str, Any] = None,
-        save_args: Dict[str, Any] = None,
-        credentials: Dict[str, Any] = None,
-        fs_args: Dict[str, Any] = None,
-        metadata: Dict[str, Any] = None,
+        load_args: dict[str, Any] = None,
+        save_args: dict[str, Any] = None,
+        credentials: dict[str, Any] = None,
+        fs_args: dict[str, Any] = None,
+        metadata: dict[str, Any] = None,
     ) -> None:
         """
         Creates a new instance of ``BioSequenceDataset`` pointing
@@ -111,20 +111,20 @@ def __init__(  # noqa: PLR0913
 
         self.metadata = metadata
 
-    def _describe(self) -> Dict[str, Any]:
+    def _describe(self) -> dict[str, Any]:
         return {
             "filepath": self._filepath,
             "protocol": self._protocol,
             "load_args": self._load_args,
             "save_args": self._save_args,
         }
 
-    def _load(self) -> List:
+    def _load(self) -> list:
         load_path = get_filepath_str(self._filepath, self._protocol)
         with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
             return list(SeqIO.parse(handle=fs_file, **self._load_args))
 
-    def _save(self, data: List) -> None:
+    def _save(self, data: list) -> None:
         save_path = get_filepath_str(self._filepath, self._protocol)
 
         with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:

kedro-datasets/kedro_datasets/dask/parquet_dataset.py

@@ -1,7 +1,7 @@
 """``ParquetDataset`` is a data set used to load and save data to parquet files using Dask
 dataframe"""
 from copy import deepcopy
-from typing import Any, Dict
+from typing import Any
 
 import dask.dataframe as dd
 import fsspec
@@ -78,18 +78,18 @@ class ParquetDataset(AbstractDataset[dd.DataFrame, dd.DataFrame]):
               col3: [[int32]]
     """
 
-    DEFAULT_LOAD_ARGS: Dict[str, Any] = {}
-    DEFAULT_SAVE_ARGS: Dict[str, Any] = {"write_index": False}
+    DEFAULT_LOAD_ARGS: dict[str, Any] = {}
+    DEFAULT_SAVE_ARGS: dict[str, Any] = {"write_index": False}
 
     def __init__(  # noqa: PLR0913
         self,
         *,
         filepath: str,
-        load_args: Dict[str, Any] = None,
-        save_args: Dict[str, Any] = None,
-        credentials: Dict[str, Any] = None,
-        fs_args: Dict[str, Any] = None,
-        metadata: Dict[str, Any] = None,
+        load_args: dict[str, Any] = None,
+        save_args: dict[str, Any] = None,
+        credentials: dict[str, Any] = None,
+        fs_args: dict[str, Any] = None,
+        metadata: dict[str, Any] = None,
     ) -> None:
         """Creates a new instance of ``ParquetDataset`` pointing to concrete
         parquet files.
@@ -123,7 +123,7 @@ def __init__(  # noqa: PLR0913
             self._save_args.update(save_args)
 
     @property
-    def fs_args(self) -> Dict[str, Any]:
+    def fs_args(self) -> dict[str, Any]:
         """Property of optional file system parameters.
 
         Returns:
@@ -133,7 +133,7 @@ def fs_args(self) -> Dict[str, Any]:
         fs_args.update(self._credentials)
         return fs_args
 
-    def _describe(self) -> Dict[str, Any]:
+    def _describe(self) -> dict[str, Any]:
         return {
             "filepath": self._filepath,
             "load_args": self._load_args,

kedro-datasets/kedro_datasets/databricks/managed_table_dataset.py

@@ -4,7 +4,7 @@
 import logging
 import re
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Optional, Union
 
 import pandas as pd
 from kedro.io.core import Version, VersionNotFoundError
@@ -33,7 +33,7 @@ class ManagedTable:
     dataframe_type: str
     primary_key: Optional[str]
     owner_group: str
-    partition_columns: Union[str, List[str]]
+    partition_columns: Union[str, list[str]]
     json_schema: StructType
 
     def __post_init__(self):
@@ -203,12 +203,12 @@ def __init__(  # noqa: PLR0913
         database: str = "default",
         write_mode: Union[str, None] = None,
         dataframe_type: str = "spark",
-        primary_key: Optional[Union[str, List[str]]] = None,
+        primary_key: Optional[Union[str, list[str]]] = None,
         version: Version = None,
         # the following parameters are used by project hooks
         # to create or update table properties
-        schema: Dict[str, Any] = None,
-        partition_columns: List[str] = None,
+        schema: dict[str, Any] = None,
+        partition_columns: list[str] = None,
         owner_group: str = None,
     ) -> None:
         """Creates a new instance of ``ManagedTableDataset``.
@@ -387,7 +387,7 @@ def _save(self, data: Union[DataFrame, pd.DataFrame]) -> None:
         elif self._table.write_mode == "append":
             self._save_append(data)
 
-    def _describe(self) -> Dict[str, str]:
+    def _describe(self) -> dict[str, str]:
         """Returns a description of the instance of ManagedTableDataset
 
         Returns:

kedro-datasets/kedro_datasets/email/message_dataset.py

@@ -8,7 +8,7 @@
 from email.parser import Parser
 from email.policy import default
 from pathlib import PurePosixPath
-from typing import Any, Dict
+from typing import Any
 
 import fsspec
 from kedro.io.core import Version, get_filepath_str, get_protocol_and_path
@@ -47,19 +47,19 @@ class EmailMessageDataset(AbstractVersionedDataset[Message, Message]):
 
     """
 
-    DEFAULT_LOAD_ARGS: Dict[str, Any] = {}
-    DEFAULT_SAVE_ARGS: Dict[str, Any] = {}
+    DEFAULT_LOAD_ARGS: dict[str, Any] = {}
+    DEFAULT_SAVE_ARGS: dict[str, Any] = {}
 
     def __init__(  # noqa: PLR0913
         self,
         *,
         filepath: str,
-        load_args: Dict[str, Any] = None,
-        save_args: Dict[str, Any] = None,
+        load_args: dict[str, Any] = None,
+        save_args: dict[str, Any] = None,
         version: Version = None,
-        credentials: Dict[str, Any] = None,
-        fs_args: Dict[str, Any] = None,
-        metadata: Dict[str, Any] = None,
+        credentials: dict[str, Any] = None,
+        fs_args: dict[str, Any] = None,
+        metadata: dict[str, Any] = None,
     ) -> None:
         """Creates a new instance of ``EmailMessageDataset`` pointing to a concrete text file
         on a specific filesystem.
@@ -140,7 +140,7 @@ def __init__(  # noqa: PLR0913
         self._fs_open_args_load = _fs_open_args_load
         self._fs_open_args_save = _fs_open_args_save
 
-    def _describe(self) -> Dict[str, Any]:
+    def _describe(self) -> dict[str, Any]:
         return {
             "filepath": self._filepath,
             "protocol": self._protocol,

kedro-datasets/kedro_datasets/geopandas/geojson_dataset.py

@@ -4,7 +4,7 @@
 """
 import copy
 from pathlib import PurePosixPath
-from typing import Any, Dict, Union
+from typing import Any, Union
 
 import fsspec
 import geopandas as gpd
@@ -15,7 +15,7 @@
 
 class GeoJSONDataset(
     AbstractVersionedDataset[
-        gpd.GeoDataFrame, Union[gpd.GeoDataFrame, Dict[str, gpd.GeoDataFrame]]
+        gpd.GeoDataFrame, Union[gpd.GeoDataFrame, dict[str, gpd.GeoDataFrame]]
     ]
 ):
     """``GeoJSONDataset`` loads/saves data to a GeoJSON file using an underlying filesystem
@@ -43,19 +43,19 @@ class GeoJSONDataset(
 
     """
 
-    DEFAULT_LOAD_ARGS: Dict[str, Any] = {}
+    DEFAULT_LOAD_ARGS: dict[str, Any] = {}
     DEFAULT_SAVE_ARGS = {"driver": "GeoJSON"}
 
     def __init__(  # noqa: PLR0913
         self,
         *,
         filepath: str,
-        load_args: Dict[str, Any] = None,
-        save_args: Dict[str, Any] = None,
+        load_args: dict[str, Any] = None,
+        save_args: dict[str, Any] = None,
         version: Version = None,
-        credentials: Dict[str, Any] = None,
-        fs_args: Dict[str, Any] = None,
-        metadata: Dict[str, Any] = None,
+        credentials: dict[str, Any] = None,
+        fs_args: dict[str, Any] = None,
+        metadata: dict[str, Any] = None,
     ) -> None:
         """Creates a new instance of ``GeoJSONDataset`` pointing to a concrete GeoJSON file
         on a specific filesystem fsspec.
@@ -120,7 +120,7 @@ def __init__(  # noqa: PLR0913
         self._fs_open_args_load = _fs_open_args_load
         self._fs_open_args_save = _fs_open_args_save
 
-    def _load(self) -> Union[gpd.GeoDataFrame, Dict[str, gpd.GeoDataFrame]]:
+    def _load(self) -> Union[gpd.GeoDataFrame, dict[str, gpd.GeoDataFrame]]:
         load_path = get_filepath_str(self._get_load_path(), self._protocol)
         with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
             return gpd.read_file(fs_file, **self._load_args)
@@ -138,7 +138,7 @@ def _exists(self) -> bool:
             return False
         return self._fs.exists(load_path)
 
-    def _describe(self) -> Dict[str, Any]:
+    def _describe(self) -> dict[str, Any]:
         return {
             "filepath": self._filepath,
             "protocol": self._protocol,