Merge pull request #12 from psrenergy/gv/dim-kwargs

Allow dimension kwargs to be passed in any order

src/binary.jl

@@ -159,9 +159,9 @@ function convert(
     metadata = reader.metadata
     writer = Quiver.Writer{to}(
         filename;
-        dimensions = metadata.dimensions,
+        dimensions = String.(metadata.dimensions),
         labels = metadata.labels,
-        time_dimension = metadata.time_dimension,
+        time_dimension = String(metadata.time_dimension),
         dimension_size = metadata.dimension_size,
         initial_date = metadata.initial_date,
     )

src/csv.jl

@@ -105,7 +105,7 @@ function _quiver_next_dimension!(reader::Quiver.Reader{csv})
     end
     (row, state) = reader.reader.next
     for (i, dim) in enumerate(reader.metadata.dimensions)
-        reader.last_dimension_read[i] = row[Symbol(dim)]
+        reader.last_dimension_read[i] = row[dim]
     end
     for (i, ts) in enumerate(reader.metadata.labels)
         reader.all_labels_data_cache[i] = row[Symbol(ts)]
@@ -135,9 +135,9 @@ function convert(
     metadata = reader.metadata
     writer = Quiver.Writer{to}(
         filename;
-        dimensions = metadata.dimensions,
+        dimensions = String.(metadata.dimensions),
         labels = metadata.labels,
-        time_dimension = metadata.time_dimension,
+        time_dimension = String(metadata.time_dimension),
         dimension_size = metadata.dimension_size,
         initial_date = metadata.initial_date,
     )

src/metadata.jl

@@ -6,9 +6,9 @@ mutable struct Metadata
     # number of dimensions columns in file
     number_of_dimensions::Int
     # names of the dimensions
-    dimensions::Vector{String}
+    dimensions::Vector{Symbol}
     # Inform which dimension represent time
-    time_dimension::String
+    time_dimension::Symbol
     # unit of the time series
     unit::String
     # Maximum index for each dimension
@@ -43,8 +43,8 @@ function Metadata(;
         frequency,
         initial_date,
         length(dimensions),
-        dimensions,
-        time_dimension,
+        Symbol.(dimensions),
+        Symbol(time_dimension),
         unit,
         dimension_size,
         length(labels),
@@ -60,10 +60,10 @@ end
 function to_toml(metadata::Metadata, filename::String)
     dict_metadata = OrderedDict(
         "version" => metadata.version,
-        "dimensions" => metadata.dimensions,
+        "dimensions" => String.(metadata.dimensions),
         "dimension_size" => metadata.dimension_size,
         "initial_date" => Dates.format(metadata.initial_date, "yyyy-mm-dd HH:MM:SS"),
-        "time_dimension" => metadata.time_dimension,
+        "time_dimension" => String(metadata.time_dimension),
         "frequency" => metadata.frequency,
         "unit" => metadata.unit,
         "labels" => metadata.labels,

src/reader.jl

@@ -53,11 +53,11 @@ mutable struct Reader{I <: Implementation, R}
 end
 
 function _build_last_dimension_read!(reader::Reader; dims...)
-    for (i, dim) in enumerate(dims)
+    for (i, dim) in enumerate(reader.metadata.dimensions)
         if reader.carrousel
-            reader.last_dimension_read[i] = mod1(dim[2], reader.metadata.dimension_size[i])
+            reader.last_dimension_read[i] = mod1(dims[dim], reader.metadata.dimension_size[i])
         else
-            reader.last_dimension_read[i] = dim[2]
+            reader.last_dimension_read[i] = dims[dim]
         end
     end
     return nothing
@@ -71,6 +71,7 @@ function _move_data_from_buffer_cache_to_data!(reader::Reader)
 end
 
 function goto!(reader::Reader; dims...)
+    validate_dimensions(reader.metadata, dims...)
     _build_last_dimension_read!(reader; dims...)
     _quiver_goto!(reader)
     _move_data_from_buffer_cache_to_data!(reader)

src/utils.jl

@@ -15,16 +15,6 @@ function rm_if_exists(filename::AbstractString, remove_if_exists::Bool)
     end
 end
 
-function _assert_dimensions_are_in_order(reader_or_writer; dims...)
-    keys_dims_to_query = keys(dims)
-    for (i, dim) in enumerate(dims)
-        if dim != Symbol.(reader_or_writer.dimensions[i])
-            error("Dimensions must be read in the order of the file. (Expected the order $(reader_or_writer.dimensions)")
-        end
-    end
-    return nothing
-end
-
 function add_extension_to_file(filename::AbstractString, ext::AbstractString)
     # This regex is to check if a file has an extension
     # https://stackoverflow.com/questions/22863973/regex-check-if-a-file-has-any-extension

src/writer.jl

@@ -15,16 +15,16 @@ mutable struct Writer{I <: Implementation, W}
     end
 end
 
-function _build_last_dimension_added!(writer::Writer, dims...)
-    for (i, dim) in enumerate(dims)
-        writer.last_dimension_added[i] = dim[2]
+function _build_last_dimension_added!(writer::Writer; dims...)
+    for (i, dim) in enumerate(writer.metadata.dimensions)
+        writer.last_dimension_added[i] = dims[dim]
     end
     return nothing
 end
 
 function write!(writer::Writer, data::Vector{T}; dims...) where T <: Real
     validate_dimensions(writer.metadata, dims...)
-    _build_last_dimension_added!(writer, dims...)
+    _build_last_dimension_added!(writer; dims...)
     _quiver_write!(writer, data)
 end
 

test/test_read_write.jl

@@ -421,10 +421,10 @@ function read_outside_bounds_1(impl)
         end
     end
 
-    @test_throws EOFError Quiver.goto!(reader; stage = num_stages+1, num_scenarios, max_num_blocks, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, scenario = num_scenarios+1, max_num_blocks, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, num_scenarios, block = max_num_blocks+1, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, num_scenarios, max_num_blocks, segment = max_num_segments+1)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages+1, scenario = num_scenarios, block = max_num_blocks, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = num_scenarios+1, block = max_num_blocks, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = num_scenarios, block = max_num_blocks+1, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = num_scenarios, block = max_num_blocks, segment = max_num_segments+1)
 
     Quiver.close!(reader)
 
@@ -490,10 +490,10 @@ function read_outside_bounds_2(impl)
         end
     end
 
-    @test_throws EOFError Quiver.goto!(reader; stage = num_stages+1, num_scenarios, max_num_blocks, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, scenario = num_scenarios+1, max_num_blocks, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, num_scenarios, block = max_num_blocks+1, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, num_scenarios, max_num_blocks, segment = max_num_segments+1)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages+1, scenario = num_scenarios, block = max_num_blocks, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = num_scenarios+1, block = max_num_blocks, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = num_scenarios, block = max_num_blocks+1, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = num_scenarios, block = max_num_blocks, segment = max_num_segments+1)
 
     Quiver.close!(reader)
 
@@ -558,10 +558,10 @@ function read_outside_bounds_3(impl)
         end
     end
 
-    @test_throws EOFError Quiver.goto!(reader; stage = num_stages+1, max_num_scenarios, num_blocks, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, scenario = max_num_scenarios+1, num_blocks, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, max_num_scenarios, block = num_blocks+1, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, max_num_scenarios, num_blocks, segment = max_num_segments+1)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages+1, scenario = max_num_scenarios, block = num_blocks, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = max_num_scenarios+1, block = num_blocks, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = max_num_scenarios, block = num_blocks+1, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = max_num_scenarios, block = num_blocks, segment = max_num_segments+1)
 
     Quiver.close!(reader)
 
@@ -625,10 +625,10 @@ function read_outside_bounds_4(impl)
         end
     end
 
-    @test_throws EOFError Quiver.goto!(reader; stage = num_stages+1, num_scenarios, num_blocks, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, scenario = num_scenarios+1, num_blocks, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, num_scenarios, block = num_blocks+1, max_num_segments)
-    @test_throws EOFError Quiver.goto!(reader; num_stages, num_scenarios, num_blocks, segment = max_num_segments+1)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages+1, scenario = num_scenarios, block = num_blocks, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = num_scenarios+1, block = num_blocks, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = num_scenarios, block = num_blocks+1, segment = max_num_segments)
+    @test_throws EOFError Quiver.goto!(reader; stage = num_stages, scenario = num_scenarios, block = num_blocks, segment = max_num_segments+1)
 
     Quiver.close!(reader)
 
@@ -723,6 +723,65 @@ function read_filtering_labels(impl)
     rm("$filename.toml")
 end
 
+function read_write_out_of_order_kwargs(impl)
+    filename = joinpath(@__DIR__, "test_read_write_out_of_order_kwargs")
+
+    initial_date = DateTime(2006, 1, 1)
+    num_stages = 10
+    dates = collect(initial_date:Dates.Month(1):initial_date + Dates.Month(num_stages - 1))
+    num_scenarios = 12
+    num_blocks_per_stage = Int32.(Dates.daysinmonth.(dates) .* 24)
+    num_time_series = 3
+
+    dimensions = ["stage", "scenario", "block"]
+    labels = ["agent_$i" for i in 1:num_time_series]
+    time_dimension = "stage"
+    dimension_size = [num_stages, num_scenarios, maximum(num_blocks_per_stage)]
+
+    writer = Quiver.Writer{impl}(
+        filename;
+        dimensions,
+        labels,
+        time_dimension,
+        dimension_size,
+        initial_date = initial_date
+    )
+
+    for stage in 1:num_stages
+        for scenario in 1:num_scenarios
+            for block in 1:num_blocks_per_stage[stage]
+                data = [stage, scenario, block]
+                Quiver.write!(writer, data; block, stage, scenario)
+            end
+        end
+    end
+
+    @test_throws ErrorException Quiver.write!(writer, [1, 1, 1]; wrong_name = 1, scenario = 1, block = 1)
+
+    Quiver.close!(writer)
+
+    reader = Quiver.Reader{impl}(filename)
+    for stage in 1:num_stages
+        for scenario in 1:num_scenarios
+            for block in 1:num_blocks_per_stage[stage]
+                if impl == Quiver.csv
+                    Quiver.next_dimension!(reader)
+                else
+                    Quiver.goto!(reader; scenario, block, stage)
+                end
+                @test reader.data == [stage, scenario, block]
+            end
+        end
+    end
+
+    @test_throws ErrorException Quiver.goto!(reader; wrong_name = 1, scenario = 1, block = 1)
+
+    Quiver.close!(reader)
+
+    rm("$filename.$(Quiver.file_extension(impl))")
+    rm("$filename.toml")
+end
+
 function binary_to_csv()
     filename = joinpath(@__DIR__, "test_binary_to_csv")
 
@@ -845,6 +904,7 @@ function test_read_write_implementations()
             read_outside_bounds_3(impl)
             read_outside_bounds_4(impl)
             read_filtering_labels(impl)
+            read_write_out_of_order_kwargs(impl)
         end
     end
     @testset "Converter" begin