From 8891b8a561220d20e95316c20253d18e6eb6d5d7 Mon Sep 17 00:00:00 2001 From: guilhermebodin Date: Thu, 12 Sep 2024 11:45:22 -0300 Subject: [PATCH] update --- Project.toml | 2 +- test/test_convert.jl | 134 +++++++++++++++++++++++++++++++ test/test_merge.jl | 88 ++++++++++++++++++++ test/test_read_write.jl | 173 ---------------------------------------- 4 files changed, 223 insertions(+), 174 deletions(-) create mode 100644 test/test_convert.jl create mode 100644 test/test_merge.jl diff --git a/Project.toml b/Project.toml index c5d05fd..52167f8 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Quiver" uuid = "cdbb3f72-2527-4dbd-9d0e-93533a5519ac" authors = ["raphasampaio", "guilhermebodin"] -version = "0.1.2" +version = "0.1.3" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" diff --git a/test/test_convert.jl b/test/test_convert.jl new file mode 100644 index 0000000..1c487bd --- /dev/null +++ b/test/test_convert.jl @@ -0,0 +1,134 @@ +module TestConvert + +using Dates +using Quiver +using Test + +function binary_to_csv() + filename = joinpath(@__DIR__, "test_binary_to_csv") + + initial_date = DateTime(2006, 1, 1) + num_stages = 10 + dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1)) + num_scenarios = 12 + num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24) + num_time_series = 3 + + dimensions = ["stage", "scenario", "block"] + labels = ["agent_$i" for i in 1:num_time_series] + time_dimension = "stage" + dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)] + + writer = Quiver.Writer{Quiver.binary}( + filename; + dimensions, + labels, + time_dimension, + dimension_size, + initial_date = initial_date + ) + + for stage in 1:num_stages + for scenario in 1:num_scenarios + for block in 1:num_blocks_per_stage[stage] + data = [stage, scenario, block] + Quiver.write!(writer, data; stage, scenario, block) + end + end + end + + Quiver.close!(writer) + + Quiver.convert(filename, Quiver.binary, Quiver.csv) + + reader = Quiver.Reader{Quiver.csv}(filename) + for stage in 1:num_stages + for scenario in 1:num_scenarios + for block in 1:num_blocks_per_stage[stage] + Quiver.next_dimension!(reader) + @test reader.data == [stage, scenario, block] + end + end + end + + Quiver.close!(reader) + + rm("$filename.$(Quiver.file_extension(Quiver.binary))") + rm("$filename.$(Quiver.file_extension(Quiver.csv))") + rm("$filename.toml") +end + +function csv_to_binary() + filename = joinpath(@__DIR__, "test_csv_to_binary") + + initial_date = DateTime(2006, 1, 1) + num_stages = 10 + dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1)) + num_scenarios = 12 + num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24) + num_time_series = 3 + + dimensions = ["stage", "scenario", "block"] + labels = ["agent_$i" for i in 1:num_time_series] + time_dimension = "stage" + dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)] + + writer = Quiver.Writer{Quiver.csv}( + filename; + dimensions, + labels, + time_dimension, + dimension_size, + initial_date = initial_date + ) + + for stage in 1:num_stages + for scenario in 1:num_scenarios + for block in 1:num_blocks_per_stage[stage] + data = [stage, scenario, block] + Quiver.write!(writer, data; stage, scenario, block) + end + end + end + + Quiver.close!(writer) + + Quiver.convert(filename, Quiver.csv, Quiver.binary) + + reader = Quiver.Reader{Quiver.binary}(filename) + for stage in 1:num_stages + for scenario in 1:num_scenarios + for block in 1:num_blocks_per_stage[stage] + Quiver.goto!(reader; stage, scenario, block) + @test reader.data == [stage, scenario, block] + end + end + end + + Quiver.close!(reader) + + rm("$filename.$(Quiver.file_extension(Quiver.csv))") + rm("$filename.$(Quiver.file_extension(Quiver.binary))") + rm("$filename.toml") +end + +function test_convert() + binary_to_csv() + csv_to_binary() +end + +function runtests() + Base.GC.gc() + Base.GC.gc() + for name in names(@__MODULE__; all = true) + if startswith("$name", "test_") + @testset "$(name)" begin + getfield(@__MODULE__, name)() + end + end + end +end + +TestConvert.runtests() + +end \ No newline at end of file diff --git a/test/test_merge.jl b/test/test_merge.jl new file mode 100644 index 0000000..30c1f71 --- /dev/null +++ b/test/test_merge.jl @@ -0,0 +1,88 @@ +module TestMerge + +using Dates +using Quiver +using Test + +function merge_files(impl) + filename = joinpath(@__DIR__, "test_read_write_merge") + num_files = 3 + filenames = ["$(filename)_$(i)" for i in 1:num_files] + + initial_date = DateTime(2006, 1, 1) + num_stages = 10 + num_scenarios = 12 + num_blocks = 24 + num_time_series = 3 + + dimensions = ["stage", "scenario", "block"] + time_dimension = "stage" + dimension_size = [num_stages, num_scenarios, num_blocks] + + for file in 1:num_files + labels = ["agent_$file"] + writer = Quiver.Writer{impl}( + filenames[file]; + dimensions, + labels, + time_dimension, + dimension_size, + initial_date = initial_date + ) + + for stage in 1:num_stages + for scenario in 1:num_scenarios + for block in 1:num_blocks + data = [stage, scenario, block + scenario][file] + Quiver.write!(writer, [data]; stage, scenario, block) + end + end + end + + Quiver.close!(writer) + end + + output_filename = joinpath(@__DIR__, "test_read_write_merge_merged") + Quiver.merge(output_filename, filenames, impl) + + reader = Quiver.Reader{impl}(output_filename) + for stage in 1:num_stages + for scenario in 1:num_scenarios + for block in 1:num_blocks + Quiver.goto!(reader; stage, scenario, block) + @test reader.data == [stage, scenario, block + scenario] + end + end + end + + Quiver.close!(reader) + + for filename in filenames + rm("$filename.$(Quiver.file_extension(impl))") + rm("$filename.toml") + end + rm("$output_filename.$(Quiver.file_extension(impl))") + rm("$output_filename.toml") +end + +function test_merge() + for impl in Quiver.implementations() + merge_files(impl) + end +end + +function runtests() + Base.GC.gc() + Base.GC.gc() + for name in names(@__MODULE__; all = true) + if startswith("$name", "test_") + @testset "$(name)" begin + getfield(@__MODULE__, name)() + end + end + end +end + +TestMerge.runtests() + +end \ No newline at end of file diff --git a/test/test_read_write.jl b/test/test_read_write.jl index 1ca4c1f..e01ce96 100644 --- a/test/test_read_write.jl +++ b/test/test_read_write.jl @@ -468,67 +468,6 @@ function read_write_goto_csv_3() rm("$filename.toml") end -function read_write_merge(impl) - filename = joinpath(@__DIR__, "test_read_write_merge") - num_files = 3 - filenames = ["$(filename)_$(i)" for i in 1:num_files] - - initial_date = DateTime(2006, 1, 1) - num_stages = 10 - num_scenarios = 12 - num_blocks = 24 - num_time_series = 3 - - dimensions = ["stage", "scenario", "block"] - time_dimension = "stage" - dimension_size = [num_stages, num_scenarios, num_blocks] - - for file in 1:num_files - labels = ["agent_$file"] - writer = Quiver.Writer{impl}( - filenames[file]; - dimensions, - labels, - time_dimension, - dimension_size, - initial_date = initial_date - ) - - for stage in 1:num_stages - for scenario in 1:num_scenarios - for block in 1:num_blocks - data = [stage, scenario, block + scenario][file] - Quiver.write!(writer, [data]; stage, scenario, block) - end - end - end - - Quiver.close!(writer) - end - - output_filename = joinpath(@__DIR__, "test_read_write_merge_merged") - Quiver.merge(output_filename, filenames, impl) - - reader = Quiver.Reader{impl}(output_filename) - for stage in 1:num_stages - for scenario in 1:num_scenarios - for block in 1:num_blocks - Quiver.goto!(reader; stage, scenario, block) - @test reader.data == [stage, scenario, block + scenario] - end - end - end - - Quiver.close!(reader) - - for filename in filenames - rm("$filename.$(Quiver.file_extension(impl))") - rm("$filename.toml") - end - rm("$output_filename.$(Quiver.file_extension(impl))") - rm("$output_filename.toml") -end - function read_write_carrousel(impl) if impl == Quiver.csv return @@ -1009,114 +948,6 @@ function read_write_out_of_order_kwargs(impl) rm("$filename.toml") end -function binary_to_csv() - filename = joinpath(@__DIR__, "test_binary_to_csv") - - initial_date = DateTime(2006, 1, 1) - num_stages = 10 - dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1)) - num_scenarios = 12 - num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24) - num_time_series = 3 - - dimensions = ["stage", "scenario", "block"] - labels = ["agent_$i" for i in 1:num_time_series] - time_dimension = "stage" - dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)] - - writer = Quiver.Writer{Quiver.binary}( - filename; - dimensions, - labels, - time_dimension, - dimension_size, - initial_date = initial_date - ) - - for stage in 1:num_stages - for scenario in 1:num_scenarios - for block in 1:num_blocks_per_stage[stage] - data = [stage, scenario, block] - Quiver.write!(writer, data; stage, scenario, block) - end - end - end - - Quiver.close!(writer) - - Quiver.convert(filename, Quiver.binary, Quiver.csv) - - reader = Quiver.Reader{Quiver.csv}(filename) - for stage in 1:num_stages - for scenario in 1:num_scenarios - for block in 1:num_blocks_per_stage[stage] - Quiver.next_dimension!(reader) - @test reader.data == [stage, scenario, block] - end - end - end - - Quiver.close!(reader) - - rm("$filename.$(Quiver.file_extension(Quiver.binary))") - rm("$filename.$(Quiver.file_extension(Quiver.csv))") - rm("$filename.toml") -end - -function csv_to_binary() - filename = joinpath(@__DIR__, "test_csv_to_binary") - - initial_date = DateTime(2006, 1, 1) - num_stages = 10 - dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1)) - num_scenarios = 12 - num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24) - num_time_series = 3 - - dimensions = ["stage", "scenario", "block"] - labels = ["agent_$i" for i in 1:num_time_series] - time_dimension = "stage" - dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)] - - writer = Quiver.Writer{Quiver.csv}( - filename; - dimensions, - labels, - time_dimension, - dimension_size, - initial_date = initial_date - ) - - for stage in 1:num_stages - for scenario in 1:num_scenarios - for block in 1:num_blocks_per_stage[stage] - data = [stage, scenario, block] - Quiver.write!(writer, data; stage, scenario, block) - end - end - end - - Quiver.close!(writer) - - Quiver.convert(filename, Quiver.csv, Quiver.binary) - - reader = Quiver.Reader{Quiver.binary}(filename) - for stage in 1:num_stages - for scenario in 1:num_scenarios - for block in 1:num_blocks_per_stage[stage] - Quiver.goto!(reader; stage, scenario, block) - @test reader.data == [stage, scenario, block] - end - end - end - - Quiver.close!(reader) - - rm("$filename.$(Quiver.file_extension(Quiver.csv))") - rm("$filename.$(Quiver.file_extension(Quiver.binary))") - rm("$filename.toml") -end - function test_read_write_implementations() for impl in Quiver.implementations() @testset "Read and Write $(impl)" begin @@ -1140,10 +971,6 @@ function test_read_write_implementations() read_write_out_of_order_kwargs(impl) end end - @testset "Converter" begin - binary_to_csv() - csv_to_binary() - end end function runtests()