feature: incremental zstd mode

ipetkov · Sep 17, 2023 · c644901 · c644901
1 parent 8b4f7a4
commit c644901
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 0 deletions.
diff --git a/lib/setupHooks/inheritCargoArtifactsHook.sh b/lib/setupHooks/inheritCargoArtifactsHook.sh
@@ -19,8 +19,15 @@ inheritCargoArtifacts() {
   if [ -f "${preparedArtifacts}" ]; then
     echo "decompressing cargo artifacts from ${preparedArtifacts} to ${cargoTargetDir}"
 
+    if [ -f "${preparedArtifacts}.prev" ]; then
+      inheritCargoArtifacts "$(readlink -f "${preparedArtifacts}.prev")" "$cargoTargetDir"
+    fi
+
     zstd -d "${preparedArtifacts}" --stdout | \
       tar -x -C "${cargoTargetDir}" --strip-components=1
+    rm -f "${cargoTargetDir}/.crane-previous-archive"
+    ln -s "${preparedArtifacts}" "${cargoTargetDir}/.crane-previous-archive"
+
   elif [ -d "${preparedArtifacts}" ]; then
     echo "copying cargo artifacts from ${preparedArtifacts} to ${cargoTargetDir}"
 

diff --git a/lib/setupHooks/installCargoArtifactsHook.sh b/lib/setupHooks/installCargoArtifactsHook.sh
@@ -18,6 +18,37 @@ compressAndInstallCargoArtifactsDir() {
   )
 }
 
+compressAndInstallCargoArtifactsDirIncremental() {
+  local dir="${1:?destination directory not defined}"
+  local cargoTargetDir="${2:?cargoTargetDir not defined}"
+
+  mkdir -p "${dir}"
+
+  local dest="${dir}/target.tar.zst"
+  echo "compressing ${cargoTargetDir} to ${dest}"
+  (
+    export SOURCE_DATE_EPOCH=1
+    touch -d @${SOURCE_DATE_EPOCH} "${TMPDIR}/.crane.source-date-epoch"
+
+    find "${cargoTargetDir}" \
+      -newer "${TMPDIR}/.crane.source-date-epoch" \
+      -print0 \
+      | tar \
+      --null \
+      --sort=name \
+      --mtime="@${SOURCE_DATE_EPOCH}" \
+      --owner=0 \
+      --group=0 \
+      --numeric-owner \
+      --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \
+      -c -f - -T - \
+      | zstd "-T${NIX_BUILD_CORES:-0}" -o "${dest}"
+     if [ -e "${cargoTargetDir}/.crane-previous-archive" ]; then
+       cp -a "${cargoTargetDir}/.crane-previous-archive" "${dest}.prev"
+     fi
+   )
+}
+
 dedupAndInstallCargoArtifactsDir() {
   local dest="${1:?destination directory not defined}"
   local cargoTargetDir="${2:?cargoTargetDir not defined}"
@@ -55,6 +86,10 @@ prepareAndInstallCargoArtifactsDir() {
 
   case "${mode}" in
     "use-zstd")
+      compressAndInstallCargoArtifactsDirIncremental "${dir}" "${cargoTargetDir}"
+      ;;
+
+    "use-zstd-no-incr")
       compressAndInstallCargoArtifactsDir "${dir}" "${cargoTargetDir}"
       ;;