diff --git a/reproducibility_tests/reproducibility_tools.jl b/reproducibility_tests/reproducibility_tools.jl index 0ecb8844b5..9bd40d8258 100644 --- a/reproducibility_tests/reproducibility_tools.jl +++ b/reproducibility_tests/reproducibility_tools.jl @@ -180,7 +180,8 @@ function reproducibility_results( :no_comparable_dirs, ) - data_file_references = map(p -> joinpath(p, reference_filename), dirs) + data_file_references = + map(p -> joinpath(p, job_id, reference_filename), dirs) # foreach(x->maybe_extract(x), data_file_references) @@ -275,8 +276,9 @@ function export_reproducibility_results( skip, ) + commit_shas = readdir(save_dir) for (computed_mse, dir) in zip(computed_mses, dirs) - commit_hash = basename(dirname(dir)) + commit_hash = commit_sha_from_dir(commit_shas, dir) computed_mse_file = joinpath(repro_dir, "computed_mse_$commit_hash.json") @@ -420,8 +422,9 @@ function report_reproducibility_results( table_data = hcat(sources, summary_statuses) PrettyTables.pretty_table(io, table_data; header, crop = :none) + n_comparisons = length(computed_mses) println(io, "Summary:") - println(io, " n_comparisons = $(length(computed_mses))") + println(io, " n_comparisons = $n_comparisons") println(io, " n_times_reproducible = $n_times_reproducible") println(io, " n_times_not_reproducible = $n_times_not_reproducible") println(io, " n_passes = $n_passes") @@ -440,7 +443,7 @@ function report_reproducibility_results( elseif test_broken_report_flakiness return :not_yet_reproducible else - if n_passes ≥ n_pass_limit + if n_passes ≥ n_pass_limit || n_passes == n_comparisons return :reproducible else return :not_reproducible diff --git a/reproducibility_tests/reproducibility_utils.jl b/reproducibility_tests/reproducibility_utils.jl index 3e08beb0bc..306e38bae4 100644 --- a/reproducibility_tests/reproducibility_utils.jl +++ b/reproducibility_tests/reproducibility_utils.jl @@ -51,10 +51,9 @@ assist our understanding and reasoning, we let's assume that there are two state ## state 2: data is saved for future reference - - `commit_hash/job_id/output_dir/` - - `commit_hash/job_id/output_dir/reproducibility_bundle/` - - `commit_hash/job_id/output_dir/reproducibility_bundle/ref_counter.jl` - - `commit_hash/job_id/output_dir/reproducibility_bundle/prog_state.hdf5` + - `commit_hash/job_id/reproducibility_bundle/` + - `commit_hash/job_id/reproducibility_bundle/ref_counter.jl` + - `commit_hash/job_id/reproducibility_bundle/prog_state.hdf5` - `commit_hash/reproducibility_bundle/ref_counter.jl` - `commit_hash/reproducibility_bundle/job_id/` @@ -458,12 +457,12 @@ print_dir_tree(dir) = print_dir_tree(stdout, dir) print_dir_tree(io::IO, dir) = println(io, string_dir_tree(dir)) function string_dir_tree(dir) - s = "Files in `$dir`\n:" + s = "Files in `$dir`:\n" for (root, _, files) in walkdir(dir) for file in files f = joinpath(root, file) isfile(f) || continue # rm symlink folders (included but not files) - s *= " $f\n" + s *= " $(replace(f, dir => ""))\n" end end return s @@ -480,8 +479,7 @@ end ref_counter_PR = read_ref_counter(ref_counter_file_PR), skip = get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) != "climaatmos-ci", dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", strip_folder = strip_output_active_path, ) @@ -519,8 +517,7 @@ function move_data_to_save_dir(; ref_counter_PR = read_ref_counter(ref_counter_file_PR), skip = get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) != "climaatmos-ci", dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", strip_folder = strip_output_active_path, ) @@ -541,36 +538,20 @@ function move_data_to_save_dir(; dirs_src, dest_root, commit, - n_hash_characters, repro_folder, strip_folder, ) if debug_reproducibility() - @show repro_folder - @show dirs_src - @show dest_root - @show files_dest - @show files_src - @show isfile.(files_src) println("******") foreach(print_dir_tree, dirs_src) println("******") - print_dir_tree(dest_root) - println("******") end for (src, dest) in zip(files_src, files_dest) - @show src - @show dest @assert isfile(src) mkpath(dirname(dest)) mv(src, dest; force = true) end - dest_repro = destination_directory(; - dest_root, - commit, - n_hash_characters, - repro_folder, - ) + dest_repro = destination_directory(; dest_root, commit, repro_folder) ref_counter_file_main = joinpath(dest_repro, "ref_counter.jl") debug_reproducibility() && @info "Repro: moving $ref_counter_file_PR to $ref_counter_file_main" @@ -590,14 +571,40 @@ function move_data_to_save_dir(; end end +""" + get_commit_sha(; + n_hash_characters = 7, + commit = get(ENV, "BUILDKITE_COMMIT", nothing) + ) + +Returns a string of the commit hash. +""" +get_commit_sha(; + n_hash_characters = 7, + commit = get(ENV, "BUILDKITE_COMMIT", nothing), +) = return commit[1:min(n_hash_characters, length(commit))] + +function commit_sha_from_dir(commit_shas, dir) + while true + if isempty(dir) + error("Unfound commit sha.") + else + b = basename(dir) + if b in commit_shas || any(x -> occursin(b, x), commit_shas) + return b + else + dir = dirname(dir) + end + end + end +end """ save_dir_transform( src; job_id, dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", strip_folder = strip_output_active_path, ) @@ -607,7 +614,6 @@ Returns the output file, to be saved, given: - `job_id` the job ID - `dest_root` the destination root directory - `commit` the commit hash - - `n_hash_characters` truncates the commit hash to given number of characters - `repro_folder` reproducibility folder - `strip_folder` function to strip folders in output path """ @@ -615,17 +621,11 @@ function save_dir_transform( src; job_id, dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", strip_folder = strip_output_active_path, ) - dest_repro = destination_directory(; - dest_root, - commit, - n_hash_characters, - repro_folder, - ) + dest_repro = destination_directory(; dest_root, commit, repro_folder) src_filename = basename(src) dst = joinpath(dest_repro, job_id, src_filename) return strip_output_active_path(dst) @@ -634,8 +634,7 @@ end """ destination_directory(; dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", ) @@ -643,17 +642,14 @@ Return the reproducibility destination directory: `root/commit_sha/repro_folder`, given: - `dest_root` the destination root directory - `commit` the commit hash - - `n_hash_characters` truncates the commit hash to given number of characters - `repro_folder` reproducibility folder """ function destination_directory(; dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", ) - commit_sha = commit[1:min(n_hash_characters, length(commit))] - return joinpath(dest_root, commit_sha, repro_folder) + return joinpath(dest_root, commit, repro_folder) end """ diff --git a/reproducibility_tests/test_mse.jl b/reproducibility_tests/test_mse.jl index 7e586b2059..029f87ee83 100644 --- a/reproducibility_tests/test_mse.jl +++ b/reproducibility_tests/test_mse.jl @@ -62,9 +62,10 @@ else @testset "Reproducibility tests" begin commit_hashes = map(x -> basename(dirname(dirname(x))), computed_mse_filenames) + computed_mses = map(x -> parse_file_json(x), computed_mse_filenames) results = report_reproducibility_results( commit_hashes, - computed_mse_filenames; + computed_mses; test_broken_report_flakiness, ) diff --git a/test/unit_reproducibility_infra.jl b/test/unit_reproducibility_infra.jl index 5afe803d3d..c8a7862d12 100644 --- a/test/unit_reproducibility_infra.jl +++ b/test/unit_reproducibility_infra.jl @@ -687,7 +687,6 @@ end make_and_cd() do dir job_id = "job_id" commit = "commit_sha" - n_hash_characters = 10 output = "output_active" strip_folder = output repro_folder = "rbundle" @@ -698,14 +697,12 @@ end dest_root = dir, job_id, commit, - n_hash_characters, repro_folder, strip_folder, ) == joinpath(dir, dst) job_id = "job_id" commit = "commit_sha" - n_hash_characters = 10 output = "output_active" strip_folder = output repro_folder = "rbundle" @@ -716,7 +713,6 @@ end dest_root = dir, job_id, commit, - n_hash_characters, repro_folder, strip_folder, ) == joinpath(dir, dst) @@ -772,7 +768,6 @@ end dirs_src = [job_id_1, job_id_2], dest_root = save_dir, commit = "commit_sha", - n_hash_characters = 10, repro_folder, strip_folder = "output_active", ) @@ -847,7 +842,6 @@ end dest_root = save_dir, buildkite_ci = true, commit = "hash_new", - n_hash_characters = length("hash_new"), branch = "unit_test_move_data_to_save_dir", in_merge_queue = true, dirs_src = [job_id_1, job_id_2], @@ -905,7 +899,6 @@ end dest_root = save_dir, buildkite_ci = true, commit = "hash_new", - n_hash_characters = length("hash_new"), branch = "unit_test_move_data_to_save_dir", in_merge_queue = true, dirs_src = [job_id_1, job_id_2], @@ -964,7 +957,6 @@ end dest_root = save_dir, buildkite_ci = true, commit = "hash_new", - n_hash_characters = length("hash_new"), branch = "unit_test_move_data_to_save_dir", in_merge_queue = true, dirs_src = [job_id_1, job_id_2], @@ -982,6 +974,41 @@ end @test isfile(joinpath(repro_dir, "ref_counter.jl")) end end +@testset "Reproducibility infrastructure: commit_sha_from_dir" begin + @test commit_sha_from_dir( + ["CH1", "CH2", "CH3"], + joinpath("a", "b", "c", "CH1", "e", "f"), + ) == "CH1" + @test commit_sha_from_dir( + ["CH1", "CH2", "CH3"], + joinpath("a", "b", "c", "CH2", "e", "f"), + ) == "CH2" + @test commit_sha_from_dir( + ["CH1", "CH2", "CH3"], + joinpath("a", "b", "c", "e", "f", "CH2"), + ) == "CH2" + @test commit_sha_from_dir( + ["CH1", "CH2", "CH3"], + joinpath("CH2", "a", "b", "c", "e", "f"), + ) == "CH2" + + @test commit_sha_from_dir( + ["CH1_xyz", "CH2_xyz", "CH3_xyz"], + joinpath("a", "b", "c", "CH1", "e", "f"), + ) == "CH1" + @test commit_sha_from_dir( + ["CH1_xyz", "CH2_xyz", "CH3_xyz"], + joinpath("a", "b", "c", "CH2", "e", "f"), + ) == "CH2" + @test commit_sha_from_dir( + ["CH1_xyz", "CH2_xyz", "CH3_xyz"], + joinpath("a", "b", "c", "e", "f", "CH2"), + ) == "CH2" + @test commit_sha_from_dir( + ["CH1_xyz", "CH2_xyz", "CH3_xyz"], + joinpath("CH2", "a", "b", "c", "e", "f"), + ) == "CH2" +end using ClimaComms using ClimaCore: Spaces, Fields, Grids, InputOutput @@ -1019,8 +1046,18 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" end end + # ## state 1: end of simulation, folder structure + # - `job_id/output_dir/` + # - `job_id/output_dir/reproducibility_bundle/` + # - `job_id/output_dir/reproducibility_bundle/ref_counter.jl` + # - `job_id/output_dir/reproducibility_bundle/prog_state.hdf5` + # ## state 2: data is saved for future reference + # - `commit_hash/job_id/reproducibility_bundle/` + # - `commit_hash/job_id/reproducibility_bundle/ref_counter.jl` + # - `commit_hash/job_id/reproducibility_bundle/prog_state.hdf5` + @testset "Reproducibility infrastructure: reproducibility_results - legacy folder structure" begin - make_and_cd() do dir + mktempdir2_cd_computed() do (save_dir, computed_dir) grid = ExtrudedCubedSphereGrid(; z_elem = 5, z_min = 0, @@ -1033,19 +1070,25 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) comms_ctx = ClimaComms.context(space) + # Folder structure: + job_id = "unit_test" + rfolder = "rbundle" + output = "output_active" + repro_dir = joinpath(computed_dir, job_id, output, rfolder) + mkpath(repro_dir) + fv = Fields.FieldVector(; x = ones(space), y = ones(space)) - file = joinpath(dir, "my_prog_state.hdf5") + file = joinpath(repro_dir, "my_prog_state.hdf5") hdfwriter = InputOutput.HDF5Writer(file, comms_ctx) InputOutput.write!(hdfwriter, fv, "Y") Base.close(hdfwriter) # Not on buildkite - job_id = "unit_test" (d, v, how) = reproducibility_results( comms_ctx; job_id, name = "Y", - save_dir = dir, + save_dir = save_dir, ref_counter_PR = 1, reference_filename = "my_prog_state.hdf5", data_file_computed = file, @@ -1066,7 +1109,7 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" comms_ctx; job_id, name = "Y", - save_dir = dir, + save_dir = save_dir, reference_filename = "my_prog_state.hdf5", ref_counter_PR = 1, data_file_computed = file, @@ -1083,26 +1126,111 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" # Successful comparison - d01 = make_ref_file_counter(1, dir, "01") - d02 = make_ref_file_counter(2, dir, "02") - d03 = make_ref_file_counter(3, dir, "03") - d04 = make_ref_file_counter(3, dir, "04") - d05 = make_ref_file_counter(3, dir, "05") - - put_data_file(d01, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d02, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d03, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d04, fv, comms_ctx; filename = "ref_prog_state.hdf5") + commit_sha_01 = "commit_hash_01" + commit_sha_02 = "commit_hash_02" + commit_sha_03 = "commit_hash_03" + commit_sha_04 = "commit_hash_04" + commit_sha_05 = "commit_hash_05" + d01 = make_ref_file_counter(1, save_dir, commit_sha_01, rfolder) + d02 = make_ref_file_counter(2, save_dir, commit_sha_02, rfolder) + d03 = make_ref_file_counter(3, save_dir, commit_sha_03, rfolder) + d04 = make_ref_file_counter(3, save_dir, commit_sha_04, rfolder) + d05 = make_ref_file_counter(3, save_dir, commit_sha_05, rfolder) + + put_data_file( + joinpath(d01, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d02, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d03, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d04, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) fv.x .= 200 fv.y .= 300 - put_data_file(d05, fv, comms_ctx; filename = "ref_prog_state.hdf5") + put_data_file( + joinpath(d05, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + + # Test folder structure + @test isfile( + joinpath( + computed_dir, + job_id, + output, + rfolder, + "my_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_01, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_02, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_03, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_04, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_05, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) - job_id = "unit_test" (d, v, how) = reproducibility_results( comms_ctx; job_id, name = "Y", - save_dir = dir, + save_dir = save_dir, ref_counter_PR = 3, reference_filename = "ref_prog_state.hdf5", data_file_computed = file, @@ -1138,18 +1266,21 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" comms_ctx = ClimaComms.context(space) fv = Fields.FieldVector(; x = ones(space), y = ones(space)) - file = joinpath( - computed_dir, - "reproducibility_bundle", - "computed_prog_state.hdf5", - ) - mkpath(dirname(file)) + + # Folder structure: + job_id = "unit_test" + rfolder = "rbundle" + output = "output_active" + repro_dir = joinpath(computed_dir, job_id, output, rfolder) + mkpath(repro_dir) + + file = joinpath(repro_dir, "computed_prog_state.hdf5") + mkpath(repro_dir) hdfwriter = InputOutput.HDF5Writer(file, comms_ctx) InputOutput.write!(hdfwriter, fv, "Y") Base.close(hdfwriter) # Not on buildkite - job_id = "unit_test" (d, v, how) = reproducibility_results( comms_ctx; job_id, @@ -1170,7 +1301,6 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" @test how == :skipped # Empty comparable dirs - job_id = "unit_test" (d, v, how) = reproducibility_results( comms_ctx; job_id, @@ -1192,19 +1322,49 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" # Successful comparison - d01 = make_ref_file_counter(1, save_dir, rbundle("01")) - d02 = make_ref_file_counter(2, save_dir, rbundle("02")) - d03 = make_ref_file_counter(3, save_dir, rbundle("03")) - d04 = make_ref_file_counter(3, save_dir, rbundle("04")) - d05 = make_ref_file_counter(3, save_dir, rbundle("05")) - - put_data_file(d01, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d02, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d03, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d04, fv, comms_ctx; filename = "ref_prog_state.hdf5") + commit_sha_01 = "commit_hash_01" + commit_sha_02 = "commit_hash_02" + commit_sha_03 = "commit_hash_03" + commit_sha_04 = "commit_hash_04" + commit_sha_05 = "commit_hash_05" + d01 = make_ref_file_counter(1, save_dir, commit_sha_01, rfolder) + d02 = make_ref_file_counter(2, save_dir, commit_sha_02, rfolder) + d03 = make_ref_file_counter(3, save_dir, commit_sha_03, rfolder) + d04 = make_ref_file_counter(3, save_dir, commit_sha_04, rfolder) + d05 = make_ref_file_counter(3, save_dir, commit_sha_05, rfolder) + + put_data_file( + joinpath(d01, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d02, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d03, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d04, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) fv.x .= 200 fv.y .= 300 - put_data_file(d05, fv, comms_ctx; filename = "ref_prog_state.hdf5") + put_data_file( + joinpath(d05, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) job_id = "unit_test" (d, v, how) = reproducibility_results( @@ -1229,6 +1389,62 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" @test d == [d05, d04, d03] @test how == :successful_comparison + + # Test folder structure + @test isfile( + joinpath( + computed_dir, + job_id, + output, + rfolder, + "computed_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_01, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_02, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_03, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_04, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_05, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) end end @@ -1246,7 +1462,12 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) comms_ctx = ClimaComms.context(space) + # Folder structure: job_id = "unit_test_export_reproducibility_results" + rfolder = "rbundle" + output = "output_active" + repro_dir = joinpath(computed_dir, job_id, output, rfolder) + mkpath(repro_dir) fv = Fields.FieldVector(; x = ones(space), y = ones(space)) @@ -1283,25 +1504,55 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" @test isempty(dirs) # Successful comparisons, legacy path configuration - d01 = make_ref_file_counter(1, save_dir, "01") - d02 = make_ref_file_counter(2, save_dir, "02") - d03 = make_ref_file_counter(3, save_dir, "03") - d04 = make_ref_file_counter(3, save_dir, "04") - d05 = make_ref_file_counter(3, save_dir, "05") - - put_data_file(d01, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d02, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d03, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d04, fv, comms_ctx; filename = "ref_prog_state.hdf5") + commit_sha_01 = "commit_hash_01" + commit_sha_02 = "commit_hash_02" + commit_sha_03 = "commit_hash_03" + commit_sha_04 = "commit_hash_04" + commit_sha_05 = "commit_hash_05" + d01 = make_ref_file_counter(1, save_dir, commit_sha_01) + d02 = make_ref_file_counter(2, save_dir, commit_sha_02) + d03 = make_ref_file_counter(3, save_dir, commit_sha_03) + d04 = make_ref_file_counter(3, save_dir, commit_sha_04) + d05 = make_ref_file_counter(3, save_dir, commit_sha_05) + + put_data_file( + joinpath(d01, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d02, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d03, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d04, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) fv.x .= 200 fv.y .= 300 - put_data_file(d05, fv, comms_ctx; filename = "ref_prog_state.hdf5") + put_data_file( + joinpath(d05, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) - @test isfile(joinpath(d01, "ref_prog_state.hdf5")) - @test isfile(joinpath(d02, "ref_prog_state.hdf5")) - @test isfile(joinpath(d03, "ref_prog_state.hdf5")) - @test isfile(joinpath(d04, "ref_prog_state.hdf5")) - @test isfile(joinpath(d05, "ref_prog_state.hdf5")) + @test isfile(joinpath(d01, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d02, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d03, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d04, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d05, job_id, "ref_prog_state.hdf5")) @test isfile(joinpath(d01, "ref_counter.jl")) @test isfile(joinpath(d02, "ref_counter.jl")) @test isfile(joinpath(d03, "ref_counter.jl")) @@ -1341,7 +1592,12 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) comms_ctx = ClimaComms.context(space) + # Folder structure: job_id = "unit_test_export_reproducibility_results" + rfolder = "rbundle" + output = "output_active" + repro_dir = joinpath(computed_dir, job_id, output, rfolder) + mkpath(repro_dir) fv = Fields.FieldVector(; x = ones(space), y = ones(space)) @@ -1359,6 +1615,7 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" n = 10, ref_counter_PR = 1, skip = true, + repro_folder = rfolder, ) @test how == :skipped @test isempty(dirs) @@ -1377,30 +1634,61 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" n = 10, ref_counter_PR = 1, skip = false, + repro_folder = rfolder, ) @test how == :no_comparable_dirs @test isempty(dirs) - # Successful comparisons, legacy path configuration - d01 = make_ref_file_counter(1, save_dir, rbundle("CH01")) - d02 = make_ref_file_counter(2, save_dir, rbundle("CH02")) - d03 = make_ref_file_counter(3, save_dir, rbundle("CH03")) - d04 = make_ref_file_counter(3, save_dir, rbundle("CH04")) - d05 = make_ref_file_counter(3, save_dir, rbundle("CH05")) - - put_data_file(d01, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d02, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d03, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d04, fv, comms_ctx; filename = "ref_prog_state.hdf5") + # Successful comparisons, legacy path configuration (no repro folder) + commit_sha_01 = "sha_01" + commit_sha_02 = "sha_02" + commit_sha_03 = "sha_03" + commit_sha_04 = "sha_04" + commit_sha_05 = "sha_05" + d01 = make_ref_file_counter(1, save_dir, commit_sha_01) + d02 = make_ref_file_counter(2, save_dir, commit_sha_02) + d03 = make_ref_file_counter(3, save_dir, commit_sha_03) + d04 = make_ref_file_counter(3, save_dir, commit_sha_04) + d05 = make_ref_file_counter(3, save_dir, commit_sha_05) + + put_data_file( + joinpath(d01, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d02, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d03, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d04, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) fv.x .= 200 fv.y .= 300 - put_data_file(d05, fv, comms_ctx; filename = "ref_prog_state.hdf5") + put_data_file( + joinpath(d05, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) - @test isfile(joinpath(d01, "ref_prog_state.hdf5")) - @test isfile(joinpath(d02, "ref_prog_state.hdf5")) - @test isfile(joinpath(d03, "ref_prog_state.hdf5")) - @test isfile(joinpath(d04, "ref_prog_state.hdf5")) - @test isfile(joinpath(d05, "ref_prog_state.hdf5")) + @test isfile(joinpath(d01, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d02, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d03, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d04, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d05, job_id, "ref_prog_state.hdf5")) @test isfile(joinpath(d01, "ref_counter.jl")) @test isfile(joinpath(d02, "ref_counter.jl")) @test isfile(joinpath(d03, "ref_counter.jl")) @@ -1420,14 +1708,21 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" n = 10, ref_counter_PR = 3, skip = false, + repro_folder = rfolder, ) @test how == :successful_comparison @test dirs == [d05, d04, d03] - repro_dir = joinpath(computed_dir, "reproducibility_bundle") + repro_dir = joinpath(computed_dir, rfolder) @test isfile(joinpath(repro_dir, "computed_prog_state.hdf5")) - @test isfile(joinpath(repro_dir, "computed_mse_CH05.json")) - @test isfile(joinpath(repro_dir, "computed_mse_CH04.json")) - @test isfile(joinpath(repro_dir, "computed_mse_CH03.json")) + @test isfile( + joinpath(repro_dir, "computed_mse_$commit_sha_05.json"), + ) + @test isfile( + joinpath(repro_dir, "computed_mse_$commit_sha_04.json"), + ) + @test isfile( + joinpath(repro_dir, "computed_mse_$commit_sha_03.json"), + ) end end end