Skip to content

Commit

Permalink
index: push: make sure cache files exist
Browse files Browse the repository at this point in the history
  • Loading branch information
efiop committed Dec 20, 2023
1 parent 9bb4236 commit 38c66f6
Showing 1 changed file with 20 additions and 2 deletions.
22 changes: 20 additions & 2 deletions src/dvc_data/index/push.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from .build import build
from .checkout import apply, compare
from .fetch import _log_missing
from .index import ObjectStorage
from .index import DataIndex, ObjectStorage

if TYPE_CHECKING:
from dvc_objects.fs import FileSystem
Expand Down Expand Up @@ -44,6 +44,22 @@ def _onerror(cache, data, failed_keys, src_path, dest_path, exc):
)


def _filter_missing(index):
ret = DataIndex()
ret.storage_map = index.storage_map

for _, entry in index.items():
try:
cache_fs, cache_path = index.storage_map.get_cache(entry)
except ValueError:
continue

if cache_fs.exists(cache_path):
ret.add(entry)

return ret


def push(
idxs,
callback: "Callback" = DEFAULT_CALLBACK,
Expand Down Expand Up @@ -83,9 +99,11 @@ def push(
failed += len(result.failed)
else:
old = build(data.path, data.fs)

existing_fs_index = _filter_missing(fs_index)
diff = compare(
old,
fs_index,
existing_fs_index,
meta_only=True,
meta_cmp_key=partial(_meta_checksum, data.fs),
)
Expand Down

0 comments on commit 38c66f6

Please sign in to comment.