From 0ac94e32acc638dd917072cae9f2c33ab62dd996 Mon Sep 17 00:00:00 2001 From: guilhermebodin Date: Tue, 17 Dec 2024 20:55:04 -0300 Subject: [PATCH] Add a function to compare files --- Project.toml | 2 +- src/reader.jl | 35 +++++++ test/test_compare_files.jl | 186 +++++++++++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 test/test_compare_files.jl diff --git a/Project.toml b/Project.toml index a285802..ff99370 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Quiver" uuid = "cdbb3f72-2527-4dbd-9d0e-93533a5519ac" authors = ["raphasampaio", "guilhermebodin"] -version = "0.1.6" +version = "0.1.7" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" diff --git a/src/reader.jl b/src/reader.jl index d42b01f..808833e 100644 --- a/src/reader.jl +++ b/src/reader.jl @@ -312,3 +312,38 @@ function file_to_df( return df end + +function compare_files( + filename1::String, + filename2::String, + implementation::Type{I}; + labels_to_read::Vector{String} = String[], + atol::Real = 1e-6, + rtol::Real = 1e-6, +)::Bool where {I <: Implementation} + data1, metadata1 = file_to_array(filename1, implementation; labels_to_read) + data2, metadata2 = file_to_array(filename2, implementation; labels_to_read) + + if metadata1 != metadata2 + return false + end + + if size(data1) != size(data2) + return false + end + + for i in eachindex(data1) + if isnan(data1[i]) && isnan(data2[i]) + continue + elseif isnan(data1[i]) && !isnan(data2[i]) + return false + elseif !isnan(data1[i]) && isnan(data2[i]) + return false + end + if !isapprox(data1[i], data2[i]; atol = atol, rtol = rtol) + return false + end + end + + return true +end \ No newline at end of file diff --git a/test/test_compare_files.jl b/test/test_compare_files.jl new file mode 100644 index 0000000..1352f6f --- /dev/null +++ b/test/test_compare_files.jl @@ -0,0 +1,186 @@ +module TestCompareFiles + +using Dates +using Quiver +using Test + +function test_compare_files_with_different_metadata() + filename1 = joinpath(@__DIR__, "test_compare_files1_different_metadata") + filename2 = joinpath(@__DIR__, "test_compare_files2_different_metadata") + + initial_date = DateTime(2006, 1, 1) + num_stages = 10 + dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1)) + num_scenarios = 12 + num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24) + num_time_series = 3 + + dimensions = ["stage", "scenario", "block"] + labels = ["agent_$i" for i in 1:num_time_series] + time_dimension = "stage" + dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)] + + writer1 = Quiver.Writer{Quiver.binary}( + filename1; + dimensions, + labels, + time_dimension, + dimension_size, + initial_date = initial_date + ) + + writer2 = Quiver.Writer{Quiver.binary}( + filename2; + dimensions, + labels, + time_dimension, + dimension_size, + initial_date = initial_date + Dates.Month(1) + ) + + for stage in 1:num_stages + for scenario in 1:num_scenarios + for block in 1:num_blocks_per_stage[stage] + data = [stage, scenario, block] + Quiver.write!(writer1, data; stage, scenario, block) + Quiver.write!(writer2, data; stage, scenario, block) + end + end + end + + Quiver.close!(writer1) + Quiver.close!(writer2) + + @test !Quiver.compare_files(filename1, filename2, Quiver.binary) + + rm("$filename1.$(Quiver.file_extension(Quiver.binary))") + rm("$filename2.$(Quiver.file_extension(Quiver.binary))") + rm("$filename1.toml") + rm("$filename2.toml") +end + +function test_compare_files_with_different_data() + filename1 = joinpath(@__DIR__, "test_compare_files1_different_data") + filename2 = joinpath(@__DIR__, "test_compare_files2_different_data") + + initial_date = DateTime(2006, 1, 1) + num_stages = 10 + dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1)) + num_scenarios = 12 + num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24) + num_time_series = 3 + + dimensions = ["stage", "scenario", "block"] + labels = ["agent_$i" for i in 1:num_time_series] + time_dimension = "stage" + dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)] + + writer1 = Quiver.Writer{Quiver.binary}( + filename1; + dimensions, + labels, + time_dimension, + dimension_size, + initial_date = initial_date + ) + + writer2 = Quiver.Writer{Quiver.binary}( + filename2; + dimensions, + labels, + time_dimension, + dimension_size, + initial_date = initial_date + ) + + for stage in 1:num_stages + for scenario in 1:num_scenarios + for block in 1:num_blocks_per_stage[stage] + data = [stage, scenario, block] + Quiver.write!(writer1, data; stage, scenario, block) + Quiver.write!(writer2, data .+ 1; stage, scenario, block) + end + end + end + + Quiver.close!(writer1) + Quiver.close!(writer2) + + @test !Quiver.compare_files(filename1, filename2, Quiver.binary) + + rm("$filename1.$(Quiver.file_extension(Quiver.binary))") + rm("$filename2.$(Quiver.file_extension(Quiver.binary))") + rm("$filename1.toml") + rm("$filename2.toml") +end + +function test_compare_equal_files() + filename1 = joinpath(@__DIR__, "test_compare_files1_equal") + filename2 = joinpath(@__DIR__, "test_compare_files2_equal") + + initial_date = DateTime(2006, 1, 1) + num_stages = 10 + dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1)) + num_scenarios = 12 + num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24) + num_time_series = 3 + + dimensions = ["stage", "scenario", "block"] + labels = ["agent_$i" for i in 1:num_time_series] + time_dimension = "stage" + dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)] + + writer1 = Quiver.Writer{Quiver.binary}( + filename1; + dimensions, + labels, + time_dimension, + dimension_size, + initial_date = initial_date + ) + + writer2 = Quiver.Writer{Quiver.binary}( + filename2; + dimensions, + labels, + time_dimension, + dimension_size, + initial_date = initial_date + ) + + for stage in 1:num_stages + for scenario in 1:num_scenarios + for block in 1:num_blocks_per_stage[stage] + data = [stage, scenario, block] + Quiver.write!(writer1, data; stage, scenario, block) + Quiver.write!(writer2, data; stage, scenario, block) + end + end + end + + Quiver.close!(writer1) + Quiver.close!(writer2) + + @test Quiver.compare_files(filename1, filename2, Quiver.binary) + + rm("$filename1.$(Quiver.file_extension(Quiver.binary))") + rm("$filename2.$(Quiver.file_extension(Quiver.binary))") + rm("$filename1.toml") + rm("$filename2.toml") +end + +function runtests() + Base.GC.gc() + Base.GC.gc() + for name in names(@__MODULE__; all = true) + if startswith("$name", "test_") + @testset "$(name)" begin + getfield(@__MODULE__, name)() + end + end + end +end + +TestCompareFiles.runtests() + +end \ No newline at end of file