Merge pull request #3493 from CliMA/ck/repro_test

Debug repro tests

reproducibility_tests/move_output.jl

@@ -17,6 +17,7 @@ move_data_to_save_dir(;
 
 if buildkite_ci && in_merge_queue
     folders = get_reference_dirs_to_delete(; root_dir = cluster_data_prefix)
+    debug_reproducibility() && @warn "Repro: deleting folders $folders"
     for f in folders
         rm(f; recursive = true, force = true)
     end

reproducibility_tests/mse_summary.jl

@@ -10,6 +10,7 @@ include(joinpath(@__DIR__, "reproducibility_test_job_ids.jl"))
 job_ids = reproducibility_test_job_ids()
 
 computed_mses = get_computed_mses(; job_ids)
+isempty(computed_mses) && @warn "No MSEs were computed"
 print_mse_summary(; mses = computed_mses)
 print_skipped_jobs(; mses = computed_mses)
 

reproducibility_tests/ref_counter.jl

@@ -1,4 +1,4 @@
-190
+191
 
 # **README**
 #
@@ -21,6 +21,9 @@
 
 #=
 
+191
+- Reproducibility infrastructure debugging.
+
 190
 - Updated to new reproducibility infrastructure.
 

reproducibility_tests/reproducibility_tools.jl

@@ -27,7 +27,10 @@ function error_if_dissimilar_dicts(dicts, dict)
     end
 end
 
-function no_comparison_error(non_existent_files)
+all_files_in_dir(dir) =
+    map((root, dirs, files) -> joinpath(root, file), walkdir(dir))
+
+function no_comparison_error(dirs, non_existent_files)
     msg = "\n\n"
     msg *= "Pull request author:\n"
     msg *= "    It seems that a new dataset,\n"
@@ -47,6 +50,10 @@ function no_comparison_error(non_existent_files)
     msg *= "  `How to merge pull requests (PR) that get approved\n"
     msg *= "   but *break* reproducibility tests`\n\n"
     msg *= "for how to merge this PR."
+    msg *= "\n\n"
+    for dir in dirs
+        msg *= "Files in dirs: $(all_files_in_dir(dir))\n"
+    end
     error(msg)
 end
 
@@ -178,7 +185,7 @@ function reproducibility_results(
     # foreach(x->maybe_extract(x), data_file_references)
 
     non_existent_files = filter(x -> !isfile(x), data_file_references)
-    isempty(non_existent_files) || no_comparison_error(non_existent_files)
+    isempty(non_existent_files) || no_comparison_error(dirs, non_existent_files)
 
     dict_computed_solution = to_dict(data_file_computed, name, comms_ctx)
     dict_reference_solutions =
@@ -253,6 +260,7 @@ function export_reproducibility_results(
     hdfwriter = InputOutput.HDF5Writer(data_file_computed, comms_ctx)
     InputOutput.write!(hdfwriter, field_vec, name)
     Base.close(hdfwriter)
+    @info "Reproducibility: File $data_file_computed exported"
 
     (dirs, computed_mses, how) = reproducibility_results(
         comms_ctx;

reproducibility_tests/reproducibility_utils.jl

@@ -41,6 +41,10 @@ comparable states
 ################################################################################
 =#
 
+# debug_reproducibility() = false
+debug_reproducibility() =
+    get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) == "climaatmos-ci"
+
 import Dates
 import OrderedCollections
 
@@ -420,9 +424,23 @@ function move_data_to_save_dir(;
         for src in dirs_src
             dst = joinpath(dest_dir, basename(src))
             mv(src, dst; force = true)
+            debug_reproducibility() &&
+                @info "Reproducibility: File $src moved to $dst"
         end
         ref_counter_file_main = joinpath(dest_dir, "ref_counter.jl")
         mv(ref_counter_file_PR, ref_counter_file_main; force = true)
+    else
+        if debug_reproducibility()
+            @warn "Repro: skipping data movement"
+            @show in_merge_queue
+            @show branch == "main"
+            @show source_has_changed(;
+                n = 1,
+                root_dir = dest_root,
+                ref_counter_PR,
+                skip,
+            )
+        end
     end
 end
 

reproducibility_tests/test_mse.jl

@@ -10,13 +10,17 @@ include(joinpath(@__DIR__, "reproducibility_tools.jl"))
 (; job_id, out_dir, test_broken_report_flakiness) =
     reproducibility_test_params()
 
+debug = true
 repro_dir = joinpath(out_dir, "reproducibility_bundle")
 computed_mse_filenames =
     map(filter(default_is_mse_file, readdir(repro_dir))) do x
         joinpath(repro_dir, x)
     end
-
 if isempty(computed_mse_filenames)
+    @warn "No reproducibility tests performed, due to non-existent comparable data."
+    debug && @show readdir(out_dir)
+    debug && @show readdir(repro_dir)
+    debug && @show filter(default_is_mse_file, readdir(repro_dir))
     dirs = latest_comparable_dirs()
     if isempty(dirs) # no comparable references
         bins = compute_bins() # all reproducible bins, may or may not be comparable
@@ -46,19 +50,21 @@ if isempty(computed_mse_filenames)
         error(msg)
     end
 else
-    commit_hashes =
-        map(x -> basename(dirname(dirname(x))), computed_mse_filenames)
-    results = report_reproducibility_results(
-        commit_hashes,
-        computed_mse_filenames;
-        test_broken_report_flakiness,
-    )
+    @testset "Reproducibility tests" begin
+        commit_hashes =
+            map(x -> basename(dirname(dirname(x))), computed_mse_filenames)
+        results = report_reproducibility_results(
+            commit_hashes,
+            computed_mse_filenames;
+            test_broken_report_flakiness,
+        )
 
-    if test_broken_report_flakiness
-        @test results == :not_yet_reproducible
-        @test_broken results == :now_reproducible
-    else
-        @test results == :reproducible
+        if test_broken_report_flakiness
+            @test results == :not_yet_reproducible
+            @test_broken results == :now_reproducible
+        else
+            @test results == :reproducible
+        end
     end
 end