From 9bc70f8d2861e837c528c3c740055a448ed5f026 Mon Sep 17 00:00:00 2001 From: Charles Kawczynski Date: Mon, 23 Dec 2024 14:41:51 -0500 Subject: [PATCH] Wrap repro tests in testset, debug --- reproducibility_tests/move_output.jl | 1 + reproducibility_tests/mse_summary.jl | 1 + reproducibility_tests/ref_counter.jl | 5 ++- .../reproducibility_tools.jl | 12 +++++-- .../reproducibility_utils.jl | 18 +++++++++++ reproducibility_tests/test_mse.jl | 32 +++++++++++-------- 6 files changed, 53 insertions(+), 16 deletions(-) diff --git a/reproducibility_tests/move_output.jl b/reproducibility_tests/move_output.jl index 5e7784dd9d..641f1e3a19 100644 --- a/reproducibility_tests/move_output.jl +++ b/reproducibility_tests/move_output.jl @@ -17,6 +17,7 @@ move_data_to_save_dir(; if buildkite_ci && in_merge_queue folders = get_reference_dirs_to_delete(; root_dir = cluster_data_prefix) + debug_reproducibility() && @warn "Repro: deleting folders $folders" for f in folders rm(f; recursive = true, force = true) end diff --git a/reproducibility_tests/mse_summary.jl b/reproducibility_tests/mse_summary.jl index aa1f1bff71..a07592b675 100644 --- a/reproducibility_tests/mse_summary.jl +++ b/reproducibility_tests/mse_summary.jl @@ -10,6 +10,7 @@ include(joinpath(@__DIR__, "reproducibility_test_job_ids.jl")) job_ids = reproducibility_test_job_ids() computed_mses = get_computed_mses(; job_ids) +isempty(computed_mses) && @warn "No MSEs were computed" print_mse_summary(; mses = computed_mses) print_skipped_jobs(; mses = computed_mses) diff --git a/reproducibility_tests/ref_counter.jl b/reproducibility_tests/ref_counter.jl index 369c2bc646..c3ecc92a7d 100644 --- a/reproducibility_tests/ref_counter.jl +++ b/reproducibility_tests/ref_counter.jl @@ -1,4 +1,4 @@ -190 +191 # **README** # @@ -21,6 +21,9 @@ #= +191 +- Reproducibility infrastructure debugging. + 190 - Updated to new reproducibility infrastructure. diff --git a/reproducibility_tests/reproducibility_tools.jl b/reproducibility_tests/reproducibility_tools.jl index 3f456197cd..10739c34b7 100644 --- a/reproducibility_tests/reproducibility_tools.jl +++ b/reproducibility_tests/reproducibility_tools.jl @@ -27,7 +27,10 @@ function error_if_dissimilar_dicts(dicts, dict) end end -function no_comparison_error(non_existent_files) +all_files_in_dir(dir) = + map((root, dirs, files) -> joinpath(root, file), walkdir(dir)) + +function no_comparison_error(dirs, non_existent_files) msg = "\n\n" msg *= "Pull request author:\n" msg *= " It seems that a new dataset,\n" @@ -47,6 +50,10 @@ function no_comparison_error(non_existent_files) msg *= " `How to merge pull requests (PR) that get approved\n" msg *= " but *break* reproducibility tests`\n\n" msg *= "for how to merge this PR." + msg *= "\n\n" + for dir in dirs + msg *= "Files in dirs: $(all_files_in_dir(dir))\n" + end error(msg) end @@ -178,7 +185,7 @@ function reproducibility_results( # foreach(x->maybe_extract(x), data_file_references) non_existent_files = filter(x -> !isfile(x), data_file_references) - isempty(non_existent_files) || no_comparison_error(non_existent_files) + isempty(non_existent_files) || no_comparison_error(dirs, non_existent_files) dict_computed_solution = to_dict(data_file_computed, name, comms_ctx) dict_reference_solutions = @@ -253,6 +260,7 @@ function export_reproducibility_results( hdfwriter = InputOutput.HDF5Writer(data_file_computed, comms_ctx) InputOutput.write!(hdfwriter, field_vec, name) Base.close(hdfwriter) + @info "Reproducibility: File $data_file_computed exported" (dirs, computed_mses, how) = reproducibility_results( comms_ctx; diff --git a/reproducibility_tests/reproducibility_utils.jl b/reproducibility_tests/reproducibility_utils.jl index 5693f9901e..c1065fc46c 100644 --- a/reproducibility_tests/reproducibility_utils.jl +++ b/reproducibility_tests/reproducibility_utils.jl @@ -41,6 +41,10 @@ comparable states ################################################################################ =# +# debug_reproducibility() = false +debug_reproducibility() = + get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) == "climaatmos-ci" + import Dates import OrderedCollections @@ -420,9 +424,23 @@ function move_data_to_save_dir(; for src in dirs_src dst = joinpath(dest_dir, basename(src)) mv(src, dst; force = true) + debug_reproducibility() && + @info "Reproducibility: File $src moved to $dst" end ref_counter_file_main = joinpath(dest_dir, "ref_counter.jl") mv(ref_counter_file_PR, ref_counter_file_main; force = true) + else + if debug_reproducibility() + @warn "Repro: skipping data movement" + @show in_merge_queue + @show branch == "main" + @show source_has_changed(; + n = 1, + root_dir = dest_root, + ref_counter_PR, + skip, + ) + end end end diff --git a/reproducibility_tests/test_mse.jl b/reproducibility_tests/test_mse.jl index d058c26113..be9f7d01b0 100644 --- a/reproducibility_tests/test_mse.jl +++ b/reproducibility_tests/test_mse.jl @@ -10,13 +10,17 @@ include(joinpath(@__DIR__, "reproducibility_tools.jl")) (; job_id, out_dir, test_broken_report_flakiness) = reproducibility_test_params() +debug = true repro_dir = joinpath(out_dir, "reproducibility_bundle") computed_mse_filenames = map(filter(default_is_mse_file, readdir(repro_dir))) do x joinpath(repro_dir, x) end - if isempty(computed_mse_filenames) + @warn "No reproducibility tests performed, due to non-existent comparable data." + debug && @show readdir(out_dir) + debug && @show readdir(repro_dir) + debug && @show filter(default_is_mse_file, readdir(repro_dir)) dirs = latest_comparable_dirs() if isempty(dirs) # no comparable references bins = compute_bins() # all reproducible bins, may or may not be comparable @@ -46,19 +50,21 @@ if isempty(computed_mse_filenames) error(msg) end else - commit_hashes = - map(x -> basename(dirname(dirname(x))), computed_mse_filenames) - results = report_reproducibility_results( - commit_hashes, - computed_mse_filenames; - test_broken_report_flakiness, - ) + @testset "Reproducibility tests" begin + commit_hashes = + map(x -> basename(dirname(dirname(x))), computed_mse_filenames) + results = report_reproducibility_results( + commit_hashes, + computed_mse_filenames; + test_broken_report_flakiness, + ) - if test_broken_report_flakiness - @test results == :not_yet_reproducible - @test_broken results == :now_reproducible - else - @test results == :reproducible + if test_broken_report_flakiness + @test results == :not_yet_reproducible + @test_broken results == :now_reproducible + else + @test results == :reproducible + end end end