Merge pull request #29 from psrenergy/gb/add_file_comparison

Add a function to compare files

Project.toml

@@ -1,7 +1,7 @@
 name = "Quiver"
 uuid = "cdbb3f72-2527-4dbd-9d0e-93533a5519ac"
 authors = ["raphasampaio", "guilhermebodin"]
-version = "0.1.6"
+version = "0.1.7"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"

src/reader.jl

@@ -312,3 +312,38 @@ function file_to_df(
 
     return df
 end
+
+function compare_files(
+    filename1::String,
+    filename2::String,
+    implementation::Type{I};
+    labels_to_read::Vector{String} = String[],
+    atol::Real = 1e-6,
+    rtol::Real = 1e-6,
+)::Bool where {I <: Implementation}
+    data1, metadata1 = file_to_array(filename1, implementation; labels_to_read)
+    data2, metadata2 = file_to_array(filename2, implementation; labels_to_read)
+
+    if metadata1 != metadata2
+        return false
+    end
+
+    if size(data1) != size(data2)
+        return false
+    end
+
+    for i in eachindex(data1)
+        if isnan(data1[i]) && isnan(data2[i])
+            continue
+        elseif isnan(data1[i]) && !isnan(data2[i])
+            return false
+        elseif !isnan(data1[i]) && isnan(data2[i])
+            return false
+        end
+        if !isapprox(data1[i], data2[i]; atol = atol, rtol = rtol)
+            return false
+        end
+    end
+
+    return true
+end

test/test_compare_files.jl

@@ -0,0 +1,186 @@
+module TestCompareFiles
+
+using Dates
+using Quiver
+using Test
+
+function test_compare_files_with_different_metadata()
+    filename1 = joinpath(@__DIR__, "test_compare_files1_different_metadata")
+    filename2 = joinpath(@__DIR__, "test_compare_files2_different_metadata")
+
+    initial_date = DateTime(2006, 1, 1)
+    num_stages = 10
+    dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1))
+    num_scenarios = 12
+    num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24)
+    num_time_series = 3
+
+    dimensions = ["stage", "scenario", "block"]
+    labels = ["agent_$i" for i in 1:num_time_series]
+    time_dimension = "stage"
+    dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)]
+
+    writer1 = Quiver.Writer{Quiver.binary}(
+        filename1;
+        dimensions,
+        labels,
+        time_dimension,
+        dimension_size,
+        initial_date = initial_date
+    )
+
+    writer2 = Quiver.Writer{Quiver.binary}(
+        filename2;
+        dimensions,
+        labels,
+        time_dimension,
+        dimension_size,
+        initial_date = initial_date + Dates.Month(1)
+    )
+
+    for stage in 1:num_stages
+        for scenario in 1:num_scenarios
+            for block in 1:num_blocks_per_stage[stage]
+                data = [stage, scenario, block]
+                Quiver.write!(writer1, data; stage, scenario, block)
+                Quiver.write!(writer2, data; stage, scenario, block)
+            end
+        end
+    end
+
+    Quiver.close!(writer1)
+    Quiver.close!(writer2)
+
+    @test !Quiver.compare_files(filename1, filename2, Quiver.binary)
+
+    rm("$filename1.$(Quiver.file_extension(Quiver.binary))")
+    rm("$filename2.$(Quiver.file_extension(Quiver.binary))")
+    rm("$filename1.toml")
+    rm("$filename2.toml")
+end
+
+function test_compare_files_with_different_data()
+    filename1 = joinpath(@__DIR__, "test_compare_files1_different_data")
+    filename2 = joinpath(@__DIR__, "test_compare_files2_different_data")
+
+    initial_date = DateTime(2006, 1, 1)
+    num_stages = 10
+    dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1))
+    num_scenarios = 12
+    num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24)
+    num_time_series = 3
+
+    dimensions = ["stage", "scenario", "block"]
+    labels = ["agent_$i" for i in 1:num_time_series]
+    time_dimension = "stage"
+    dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)]
+
+    writer1 = Quiver.Writer{Quiver.binary}(
+        filename1;
+        dimensions,
+        labels,
+        time_dimension,
+        dimension_size,
+        initial_date = initial_date
+    )
+
+    writer2 = Quiver.Writer{Quiver.binary}(
+        filename2;
+        dimensions,
+        labels,
+        time_dimension,
+        dimension_size,
+        initial_date = initial_date
+    )
+
+    for stage in 1:num_stages
+        for scenario in 1:num_scenarios
+            for block in 1:num_blocks_per_stage[stage]
+                data = [stage, scenario, block]
+                Quiver.write!(writer1, data; stage, scenario, block)
+                Quiver.write!(writer2, data .+ 1; stage, scenario, block)
+            end
+        end
+    end
+
+    Quiver.close!(writer1)
+    Quiver.close!(writer2)
+
+    @test !Quiver.compare_files(filename1, filename2, Quiver.binary)
+
+    rm("$filename1.$(Quiver.file_extension(Quiver.binary))")
+    rm("$filename2.$(Quiver.file_extension(Quiver.binary))")
+    rm("$filename1.toml")
+    rm("$filename2.toml")
+end
+
+function test_compare_equal_files()
+    filename1 = joinpath(@__DIR__, "test_compare_files1_equal")
+    filename2 = joinpath(@__DIR__, "test_compare_files2_equal")
+
+    initial_date = DateTime(2006, 1, 1)
+    num_stages = 10
+    dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1))
+    num_scenarios = 12
+    num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24)
+    num_time_series = 3
+
+    dimensions = ["stage", "scenario", "block"]
+    labels = ["agent_$i" for i in 1:num_time_series]
+    time_dimension = "stage"
+    dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)]
+
+    writer1 = Quiver.Writer{Quiver.binary}(
+        filename1;
+        dimensions,
+        labels,
+        time_dimension,
+        dimension_size,
+        initial_date = initial_date
+    )
+
+    writer2 = Quiver.Writer{Quiver.binary}(
+        filename2;
+        dimensions,
+        labels,
+        time_dimension,
+        dimension_size,
+        initial_date = initial_date
+    )
+
+    for stage in 1:num_stages
+        for scenario in 1:num_scenarios
+            for block in 1:num_blocks_per_stage[stage]
+                data = [stage, scenario, block]
+                Quiver.write!(writer1, data; stage, scenario, block)
+                Quiver.write!(writer2, data; stage, scenario, block)
+            end
+        end
+    end
+
+    Quiver.close!(writer1)
+    Quiver.close!(writer2)
+
+    @test Quiver.compare_files(filename1, filename2, Quiver.binary)
+
+    rm("$filename1.$(Quiver.file_extension(Quiver.binary))")
+    rm("$filename2.$(Quiver.file_extension(Quiver.binary))")
+    rm("$filename1.toml")
+    rm("$filename2.toml")
+end
+
+function runtests()
+    Base.GC.gc()
+    Base.GC.gc()
+    for name in names(@__MODULE__; all = true)
+        if startswith("$name", "test_")
+            @testset "$(name)" begin
+                getfield(@__MODULE__, name)()
+            end
+        end
+    end
+end
+
+TestCompareFiles.runtests()
+
+end