Skip to content

Commit

Permalink
push: fix cloud versioning
Browse files Browse the repository at this point in the history
Parity with old functionality. Unfortunately using old mechanisms
like `latest_only`, but that will be tweaked in the future.
  • Loading branch information
efiop committed Nov 13, 2023
1 parent 9db4a3e commit 7c877cc
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 3 deletions.
3 changes: 2 additions & 1 deletion src/dvc_data/index/checkout.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,11 +247,12 @@ def meta_cmp_key(meta):
return meta
return (meta.isdir, meta.isexec)

kwargs.setdefault("meta_cmp_key", meta_cmp_key)

for change in idiff(
old,
new,
with_unchanged=relink,
meta_cmp_key=meta_cmp_key,
callback=callback,
**kwargs,
):
Expand Down
23 changes: 21 additions & 2 deletions src/dvc_data/index/push.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import logging
from typing import TYPE_CHECKING, Optional
from functools import partial
from typing import TYPE_CHECKING, Any, Optional

from dvc_objects.fs.callbacks import DEFAULT_CALLBACK

Expand All @@ -12,12 +13,24 @@
from .index import ObjectStorage

if TYPE_CHECKING:
from dvc_objects.fs import FileSystem
from dvc_objects.fs.callbacks import Callback

from dvc_data.hashfile.meta import Meta


logger = logging.getLogger(__name__)


# for files, if our version's checksum (etag) matches the latest remote
# checksum, we do not need to push, even if the version IDs don't match
def _meta_checksum(fs: "FileSystem", meta: "Meta") -> Any:
if not meta or meta.isdir:
return meta
assert fs.PARAM_CHECKSUM
return getattr(meta, fs.PARAM_CHECKSUM)


def push(
idxs,
callback: "Callback" = DEFAULT_CALLBACK,
Expand Down Expand Up @@ -57,12 +70,18 @@ def push(
failed += len(result.failed)
else:
old = build(data.path, data.fs)
diff = compare(old, fs_index)
diff = compare(
old,
fs_index,
meta_only=True,
meta_cmp_key=partial(_meta_checksum, data.fs),
)
data.fs.makedirs(data.fs.path.parent(data.path), exist_ok=True)
apply(
diff,
data.path,
data.fs,
latest_only=False,
update_meta=False,
storage="cache",
jobs=jobs,
Expand Down

0 comments on commit 7c877cc

Please sign in to comment.