Fixes

reproducibility_tests/reproducibility_utils.jl

@@ -377,6 +377,20 @@ function source_checksum(dir = pwd())
     return hash(joined_contents)
 end
 
+
+"""
+    source_has_changed(
+        n = 5,
+        root_dir = "/central/scratch/esm/slurm-buildkite/climaatmos-main",
+        ref_counter_PR = read_ref_counter(joinpath(@__DIR__, "ref_counter.jl")),
+        skip = get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) != "climaatmos-ci",
+        src_dir = dirname(@__DIR__),
+    )
+
+Returns a Boolean indicating if the `.jl` files in `src_dir` have changed base
+on `latest_comparable_dirs` (please see the argument list in the
+`latest_comparable_dirs` documentation).
+"""
 function source_has_changed(;
     n = 5,
     root_dir = "/central/scratch/esm/slurm-buildkite/climaatmos-main",
@@ -403,6 +417,19 @@ end
 rm_folder(path; strip_folder) =
     joinpath(filter(x -> !occursin(strip_folder, x), splitpath(path))...)
 
+print_dir_tree(dir) = print_dir_tree(stdout, dir)
+print_dir_tree(io::IO, dir) = println(io, string_dir_tree(dir))
+
+function string_dir_tree(dir)
+    s = "Files in $dir"
+    for (root, _, files) in walkdir(dir)
+        for file in files
+            s *= "  $(joinpath(root, file))\n"
+        end
+    end
+    return s
+end
+
 """
     move_data_to_save_dir(;
         buildkite_ci = get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) ==
@@ -479,6 +506,16 @@ function move_data_to_save_dir(;
             repro_folder,
             strip_folder,
         )
+        if debug_reproducibility() && any(x -> !isfile(x), files_src)
+            @show files_dest
+            @show files_src
+            @show isfile.(files_src)
+            println("******")
+            foreach(print_dir_tree, dirs_src)
+            println("******")
+            print_dir_tree(dest_root)
+            println("******")
+        end
         for (src, dest) in zip(files_src, files_dest)
             @assert isfile(src)
             mkpath(dirname(dest))
@@ -551,7 +588,19 @@ function save_dir_transform(
 end
 
 """
-    destination_directory()
+    destination_directory(;
+        dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main",
+        commit = get(ENV, "BUILDKITE_COMMIT", nothing),
+        n_hash_characters = 7,
+        repro_folder = "reproducibility_bundle",
+    )
+
+Return the reproducibility destination directory:
+`root/commit_sha/repro_folder`, given:
+ - `dest_root` the destination root directory
+ - `commit` the commit hash
+ - `n_hash_characters` truncates the commit hash to given number of characters
+ - `repro_folder` reproducibility folder
 """
 function destination_directory(;
     dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main",
@@ -560,8 +609,7 @@ function destination_directory(;
     repro_folder = "reproducibility_bundle",
 )
     commit_sha = commit[1:min(n_hash_characters, length(commit))]
-    dest_dir = joinpath(dest_root, commit_sha)
-    return joinpath(dest_dir, repro_folder)
+    return joinpath(dest_root, commit_sha, repro_folder)
 end
 
 """