Skip to content

Commit

Permalink
Cache Manifest files (apache#787)
Browse files Browse the repository at this point in the history
* cache manifests

* update API

* small fix

* move cache to module level

* update signature and check
  • Loading branch information
chinmay-bhat authored Sep 10, 2024
1 parent b593375 commit 1971fcf
Showing 1 changed file with 11 additions and 3 deletions.
14 changes: 11 additions & 3 deletions pyiceberg/table/snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import time
from collections import defaultdict
from enum import Enum
from functools import lru_cache
from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Mapping, Optional

from pydantic import Field, PrivateAttr, model_serializer
Expand Down Expand Up @@ -230,6 +231,13 @@ def __eq__(self, other: Any) -> bool:
)


@lru_cache
def _manifests(io: FileIO, manifest_list: str) -> List[ManifestFile]:
"""Return the manifests from the manifest list."""
file = io.new_input(manifest_list)
return list(read_manifest_list(file))


class Snapshot(IcebergBaseModel):
snapshot_id: int = Field(alias="snapshot-id")
parent_snapshot_id: Optional[int] = Field(alias="parent-snapshot-id", default=None)
Expand All @@ -250,9 +258,9 @@ def __str__(self) -> str:
return result_str

def manifests(self, io: FileIO) -> List[ManifestFile]:
if self.manifest_list is not None:
file = io.new_input(self.manifest_list)
return list(read_manifest_list(file))
"""Return the manifests for the given snapshot."""
if self.manifest_list:
return _manifests(io, self.manifest_list)
return []


Expand Down

0 comments on commit 1971fcf

Please sign in to comment.