Merge pull request #2 from CRREL/pr/msmitherdc/1

Pr/msmitherdc/1

HISTORY.md

@@ -1,9 +1,21 @@
 # cloudpathlib Changelog
 
-## v0.14.0 (Unreleased)
+## v0.15.1 (2023-07-12)
+
+- Compatibility with pydantic >= 2.0.0. ([PR #349](https://github.com/drivendataorg/cloudpathlib/pull/349))
+
+## v0.15.0 (2023-06-16)
+
+- Changed return type for `CloudPathMeta.__call__` to fix problems with pyright/pylance ([PR #330](https://github.com/drivendataorg/cloudpathlib/pull/330))
+- Make `CloudPath.is_valid_cloudpath` a TypeGuard so that type checkers can know the subclass if `is_valid_cloudpath` is called ([PR #337](https://github.com/drivendataorg/cloudpathlib/pull/337))
+- Added `follow_symlinks` to `stat` for 3.11.4 compatibility (see [bpo 39906](https://github.com/python/cpython/issues/84087))
+- Add `follow_symlinks` to `is_dir` implementation for CPython `glob` compatibility (see [CPython PR #104512](https://github.com/python/cpython/pull/104512))
+
+## v0.14.0 (2023-05-13)
 
 - Changed to pyproject.toml-based build.
 - Changed type hints from custom type variable `DerivedCloudPath` to [`typing.Self`](https://docs.python.org/3/library/typing.html#typing.Self) ([PEP 673](https://docs.python.org/3/library/typing.html#typing.Self)). This adds a dependency on the [typing-extensions](https://pypi.org/project/typing-extensions/) backport package from Python versions lower than 3.11.
+- Fixed a runtime key error when an S3 object does not have the `Content-Type` metadata set. ([Issue #331](https://github.com/drivendataorg/cloudpathlib/issues/331), [PR #332](https://github.com/drivendataorg/cloudpathlib/pull/332))
 
 ## v0.13.0 (2023-02-15)
 

cloudpathlib/anypath.py

@@ -1,7 +1,7 @@
 import os
 from abc import ABC
 from pathlib import Path
-from typing import Union
+from typing import Any, Union
 
 from .cloudpath import InvalidPrefixError, CloudPath
 from .exceptions import AnyPathTypeError
@@ -30,6 +30,32 @@ def __new__(cls, *args, **kwargs) -> Union[CloudPath, Path]:  # type: ignore
                     f"Path exception: {repr(path_exception)}"
                 )
 
+    # ===========  pydantic integration special methods ===============
+    @classmethod
+    def __get_pydantic_core_schema__(cls, _source_type: Any, _handler):
+        """Pydantic special method. See
+        https://docs.pydantic.dev/2.0/usage/types/custom/"""
+        try:
+            from pydantic_core import core_schema
+
+            return core_schema.no_info_after_validator_function(
+                cls.validate,
+                core_schema.any_schema(),
+            )
+        except ImportError:
+            return None
+
+    @classmethod
+    def validate(cls, v: str) -> Union[CloudPath, Path]:
+        """Pydantic special method. See
+        https://docs.pydantic.dev/2.0/usage/types/custom/"""
+        try:
+            return cls.__new__(cls, v)
+        except AnyPathTypeError as e:
+            # type errors no longer converted to validation errors
+            #  https://docs.pydantic.dev/2.0/migration/#typeerror-is-no-longer-converted-to-validationerror-in-validators
+            raise ValueError(e)
+
     @classmethod
     def __get_validators__(cls):
         """Pydantic special method. See

cloudpathlib/cloudpath.py

@@ -36,6 +36,10 @@
 from urllib.parse import urlparse
 from warnings import warn
 
+if sys.version_info >= (3, 10):
+    from typing import TypeGuard
+else:
+    from typing_extensions import TypeGuard
 if sys.version_info >= (3, 11):
     from typing import Self
 else:
@@ -99,10 +103,12 @@ def path_class(self) -> Type["CloudPath"]:
 implementation_registry: Dict[str, CloudImplementation] = defaultdict(CloudImplementation)
 
 
-def register_path_class(key: str) -> Callable:
-    T = TypeVar("T", bound=Type[CloudPath])
+T = TypeVar("T")
+CloudPathT = TypeVar("CloudPathT", bound="CloudPath")
+
 
-    def decorator(cls: Type[T]) -> Type[T]:
+def register_path_class(key: str) -> Callable[[Type[CloudPathT]], Type[CloudPathT]]:
+    def decorator(cls: Type[CloudPathT]) -> Type[CloudPathT]:
         if not issubclass(cls, CloudPath):
             raise TypeError("Only subclasses of CloudPath can be registered.")
         implementation_registry[key]._path_class = cls
@@ -113,34 +119,47 @@ def decorator(cls: Type[T]) -> Type[T]:
 
 
 class CloudPathMeta(abc.ABCMeta):
-    def __call__(cls, cloud_path, *args, **kwargs):
+    @overload
+    def __call__(cls: Type[T], cloud_path: CloudPathT, *args: Any, **kwargs: Any) -> CloudPathT:
+        ...
+
+    @overload
+    def __call__(
+        cls: Type[T], cloud_path: Union[str, "CloudPath"], *args: Any, **kwargs: Any
+    ) -> T:
+        ...
+
+    def __call__(
+        cls: Type[T], cloud_path: Union[str, CloudPathT], *args: Any, **kwargs: Any
+    ) -> Union[T, "CloudPath", CloudPathT]:
         # cls is a class that is the instance of this metaclass, e.g., CloudPath
+        if not issubclass(cls, CloudPath):
+            raise TypeError(
+                f"Only subclasses of {CloudPath.__name__} can be instantiated from its meta class."
+            )
 
         # Dispatch to subclass if base CloudPath
-        if cls == CloudPath:
+        if cls is CloudPath:
             for implementation in implementation_registry.values():
                 path_class = implementation._path_class
                 if path_class is not None and path_class.is_valid_cloudpath(
                     cloud_path, raise_on_error=False
                 ):
                     # Instantiate path_class instance
-                    new_obj = path_class.__new__(path_class, cloud_path, *args, **kwargs)
-                    if isinstance(new_obj, path_class):
-                        path_class.__init__(new_obj, cloud_path, *args, **kwargs)
+                    new_obj = object.__new__(path_class)
+                    path_class.__init__(new_obj, cloud_path, *args, **kwargs)  # type: ignore[type-var]
                     return new_obj
-            valid = [
+            valid_prefixes = [
                 impl._path_class.cloud_prefix
                 for impl in implementation_registry.values()
                 if impl._path_class is not None
             ]
             raise InvalidPrefixError(
-                f"Path {cloud_path} does not begin with a known prefix {valid}."
+                f"Path {cloud_path} does not begin with a known prefix {valid_prefixes}."
             )
 
-        # Otherwise instantiate as normal
-        new_obj = cls.__new__(cls, cloud_path, *args, **kwargs)
-        if isinstance(new_obj, cls):
-            cls.__init__(new_obj, cloud_path, *args, **kwargs)
+        new_obj = object.__new__(cls)
+        cls.__init__(new_obj, cloud_path, *args, **kwargs)  # type: ignore[type-var]
         return new_obj
 
     def __init__(cls, name: str, bases: Tuple[type, ...], dic: Dict[str, Any]) -> None:
@@ -244,8 +263,20 @@ def _no_prefix(self) -> str:
     def _no_prefix_no_drive(self) -> str:
         return self._str[len(self.cloud_prefix) + len(self.drive) :]
 
+    @overload
+    @classmethod
+    def is_valid_cloudpath(cls, path: "CloudPath", raise_on_error: bool = ...) -> TypeGuard[Self]:
+        ...
+
+    @overload
+    @classmethod
+    def is_valid_cloudpath(cls, path: str, raise_on_error: bool = ...) -> bool:
+        ...
+
     @classmethod
-    def is_valid_cloudpath(cls, path: Union[str, "CloudPath"], raise_on_error=False) -> bool:
+    def is_valid_cloudpath(
+        cls, path: Union[str, "CloudPath"], raise_on_error: bool = False
+    ) -> Union[bool, TypeGuard[Self]]:
         valid = str(path).lower().startswith(cls.cloud_prefix.lower())
 
         if raise_on_error and not valid:
@@ -443,11 +474,13 @@ def iterdir(self) -> Generator[Self, None, None]:
 
     def open(
         self,
-            mode="r",
-            buffering=-1,
-            encoding=None,
-            errors=None,
-            newline=None,
+            mode: str = "r",
+            buffering: int = -1,
+            encoding: Optional[str] = None,
+            errors: Optional[str] = None,
+            newline: Optional[str] = None,
+            force_overwrite_from_cloud: bool = False,  # extra kwarg not in pathlib
+            force_overwrite_to_cloud: bool = False,  # extra kwarg not in pathlib
             closefd=True,
             opener=None,
             ignore_ext=False,
@@ -737,7 +770,7 @@ def _dispatch_to_local_cache_path(self, func: str, *args, **kwargs) -> Any:
         else:
             return path_version
 
-    def stat(self) -> os.stat_result:
+    def stat(self, follow_symlinks: bool = True) -> os.stat_result:
         """Note: for many clients, we may want to override so we don't incur
         network costs since many of these properties are available as
         API calls.
@@ -746,7 +779,7 @@ def stat(self) -> os.stat_result:
             f"stat not implemented as API call for {self.__class__} so file must be downloaded to "
             f"calculate stats; this may take a long time depending on filesize"
         )
-        return self._dispatch_to_local_cache_path("stat")
+        return self._dispatch_to_local_cache_path("stat", follow_symlinks=follow_symlinks)
 
     # ===========  public cloud methods, not in pathlib ===============
     def download_to(self, destination: Union[str, os.PathLike]) -> Path:
@@ -1063,6 +1096,26 @@ def _upload_file_to_cloud(
         )
 
     # ===========  pydantic integration special methods ===============
+    @classmethod
+    def __get_pydantic_core_schema__(cls, _source_type: Any, _handler):
+        """Pydantic special method. See
+        https://docs.pydantic.dev/2.0/usage/types/custom/"""
+        try:
+            from pydantic_core import core_schema
+
+            return core_schema.no_info_after_validator_function(
+                cls.validate,
+                core_schema.any_schema(),
+            )
+        except ImportError:
+            return None
+
+    @classmethod
+    def validate(cls, v: str) -> Self:
+        """Used as a Pydantic validator. See
+        https://docs.pydantic.dev/2.0/usage/types/custom/"""
+        return cls(v)
+
     @classmethod
     def __get_validators__(cls) -> Generator[Callable[[Any], Self], None, None]:
         """Pydantic special method. See
@@ -1133,7 +1186,7 @@ def __init__(
     def __repr__(self) -> str:
         return "/".join(self._parents + [self.name])
 
-    def is_dir(self) -> bool:
+    def is_dir(self, follow_symlinks: bool = False) -> bool:
         return self._all_children is not None
 
     def exists(self) -> bool:

cloudpathlib/s3/s3client.py

@@ -142,7 +142,7 @@ def _get_metadata(self, cloud_path: S3Path) -> Dict[str, Any]:
             "last_modified": data["LastModified"],
             "size": data["ContentLength"],
             "etag": data["ETag"],
-            "content_type": data["ContentType"],
+            "content_type": data.get("ContentType", None),
             "extra": data["Metadata"],
         }
 

docs/mkdocs.yml

@@ -43,6 +43,7 @@ nav:
           - cloudpathlib.local: "api-reference/local.md"
 
 markdown_extensions:
+  - admonition
   - pymdownx.highlight
   - pymdownx.superfences
   - toc:

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "cloudpathlib"
-version = "0.14.0"
+version = "0.15.1"
 description = "pathlib-style classes for cloud storage services."
 readme = "README.md"
 authors = [{ name = "DrivenData", email = "[email protected]" }]

tests/mock_clients/mock_s3.py

@@ -90,6 +90,9 @@ def copy_from(self, CopySource=None, Metadata=None, MetadataDirective=None):
 
     def download_file(self, to_path, Config=None, ExtraArgs=None):
         to_path = Path(to_path)
+
+        to_path.parent.mkdir(parents=True, exist_ok=True)
+
         to_path.write_bytes(self.path.read_bytes())
         # track config to make sure it's used in tests
         self.resource.download_config = Config