Merge pull request #3507 from CliMA/ck/clean_repro_log2

Cleanup repro infra log, set job repro test to flaky

.buildkite/pipeline.yml

@@ -693,7 +693,7 @@ steps:
           --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml
           --job_id diagnostic_edmfx_aquaplanet
 
-          julia --color=yes --project=examples reproducibility_tests/test_mse.jl
+          julia --color=yes --project=examples reproducibility_tests/test_mse.jl --test_broken_report_flakiness true
           --job_id diagnostic_edmfx_aquaplanet
           --out_dir diagnostic_edmfx_aquaplanet/output_active
         artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*"

reproducibility_tests/reproducibility_tools.jl

@@ -203,11 +203,6 @@ function reproducibility_results(
                 dict_reference = dict_reference_solution,
             )
         end
-    if debug_reproducibility()
-        println("------ end of reproducibility_results")
-        @show computed_mses
-        println("------")
-    end
     return (dirs, computed_mses, :successful_comparison)
 end
 
@@ -381,11 +376,6 @@ function report_reproducibility_results(
 
     for computed_mse in computed_mses
         all_reproducible = true
-        if debug_reproducibility()
-            println("---- in report_reproducibility_results")
-            @show computed_mse
-            println("----")
-        end
         for (var, reproducible) in CRT.test_mse(; computed_mse)
             if !reproducible
                 all_reproducible = false
@@ -406,6 +396,7 @@ function report_reproducibility_results(
     data_scales = cflatten(map(computed_mses) do computed_mse
         collect(values(computed_mse))
     end)
+    data_scales = map(x -> :not_yet_exported, data_scales)
     statuses = cflatten(map(computed_mses) do computed_mse
         collect(values(CRT.test_mse(; computed_mse)))
     end)

reproducibility_tests/test_mse.jl

@@ -62,11 +62,6 @@ else
         commit_hashes =
             map(x -> commit_sha_from_mse_file(x), computed_mse_files)
         computed_mses = map(x -> parse_file(x), computed_mse_files)
-        if debug_reproducibility()
-            println("------ in test_mse.jl")
-            @show computed_mses
-            println("------")
-        end
         results = report_reproducibility_results(
             commit_hashes,
             computed_mses;