diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 34c6e5b893..3b9a7b267c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -693,7 +693,7 @@ steps: --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml --job_id diagnostic_edmfx_aquaplanet - julia --color=yes --project=examples reproducibility_tests/test_mse.jl + julia --color=yes --project=examples reproducibility_tests/test_mse.jl --test_broken_report_flakiness true --job_id diagnostic_edmfx_aquaplanet --out_dir diagnostic_edmfx_aquaplanet/output_active artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*" diff --git a/reproducibility_tests/reproducibility_tools.jl b/reproducibility_tests/reproducibility_tools.jl index 76a690674c..a94f550502 100644 --- a/reproducibility_tests/reproducibility_tools.jl +++ b/reproducibility_tests/reproducibility_tools.jl @@ -203,11 +203,6 @@ function reproducibility_results( dict_reference = dict_reference_solution, ) end - if debug_reproducibility() - println("------ end of reproducibility_results") - @show computed_mses - println("------") - end return (dirs, computed_mses, :successful_comparison) end @@ -381,11 +376,6 @@ function report_reproducibility_results( for computed_mse in computed_mses all_reproducible = true - if debug_reproducibility() - println("---- in report_reproducibility_results") - @show computed_mse - println("----") - end for (var, reproducible) in CRT.test_mse(; computed_mse) if !reproducible all_reproducible = false @@ -406,6 +396,7 @@ function report_reproducibility_results( data_scales = cflatten(map(computed_mses) do computed_mse collect(values(computed_mse)) end) + data_scales = map(x -> :not_yet_exported, data_scales) statuses = cflatten(map(computed_mses) do computed_mse collect(values(CRT.test_mse(; computed_mse))) end) diff --git a/reproducibility_tests/test_mse.jl b/reproducibility_tests/test_mse.jl index 06ec3be6f8..2baef20530 100644 --- a/reproducibility_tests/test_mse.jl +++ b/reproducibility_tests/test_mse.jl @@ -62,11 +62,6 @@ else commit_hashes = map(x -> commit_sha_from_mse_file(x), computed_mse_files) computed_mses = map(x -> parse_file(x), computed_mse_files) - if debug_reproducibility() - println("------ in test_mse.jl") - @show computed_mses - println("------") - end results = report_reproducibility_results( commit_hashes, computed_mses;