Skip to content

Commit

Permalink
Add more comments in README
Browse files Browse the repository at this point in the history
  • Loading branch information
guilhermebodin committed Sep 23, 2024
1 parent 803a232 commit 3e2eab8
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 5 deletions.
18 changes: 13 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@ Quiver is not the fastest data-structure for time series data, but it is designe
is to have a set of dimensions that can be used to index the data and a set of values from the time serires attributes. This allows to have a
table-like data-structure that can be used to store time series data.

Files that follow the Quiver implementation can be stored in any format that maps directly to a table-like structure with metadata.
CSV files are implemented in a way that the first few lines are used to store the metadata and the rest of the file is used to store the data., i.e.
Files that follow the Quiver implementation can be stored in any format that maps directly to a table-like structure with metadata. The metadata stores the frequency of the time series, the initial date, the unit of the data, the number of the dimension, the maximum value of each dimension, the time dimension and the version of the file.

```csv
The matadata is always stored in a TOML file in the following format:

```toml
version = 1
dimensions = ["stage", "scenario", "block"]
dimension_size = [10, 12, 744]
Expand All @@ -18,11 +19,18 @@ time_dimension = "stage"
frequency = "month"
unit = ""
labels = ["agent_1", "agent_2", "agent_3"]
---
```

And the data is stored in a csv or binary file that contains the values of the time series. The csv format is as follows:
```csv
stage,scenario,block,agent_1,agent_2,agent_3
1,1,1,1.0,1.0,1.0
1,1,2,1.0,1.0,1.0
1,1,3,1.0,1.0,1.0
```

The metadata stores the frequency of the time series, the initial date, the unit of the data, the number of the dimension, the maximum value of each dimension, the time dimension and the version of the file.
## Installation

```julia
pkg> add Quiver
```
32 changes: 32 additions & 0 deletions src/reader.jl
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,14 @@ function _move_data_from_buffer_cache_to_data!(reader::Reader)
return nothing
end

"""
goto!(
reader::Reader;
dims...
)
Move the reader to the specified dimensions and return the data.
"""
function goto!(reader::Reader; dims...)
validate_dimensions(reader.metadata, dims...)
_build_dimension_to_read!(reader; dims...)
Expand All @@ -89,12 +97,22 @@ function goto!(reader::Reader; dims...)
return reader.data
end

"""
next_dimension!(reader::Reader)
Move the reader to the next dimension and return the data.
"""
function next_dimension!(reader::Reader)
_quiver_next_dimension!(reader)
_move_data_from_buffer_cache_to_data!(reader)
return reader.data
end

"""
max_index(reader::Reader, dimension::String)
Return the maximum index of the specified dimension.
"""
function max_index(reader::Reader, dimension::String)
symbol_dim = Symbol(dimension)
index = findfirst(isequal(symbol_dim), reader.metadata.dimensions)
Expand All @@ -104,6 +122,11 @@ function max_index(reader::Reader, dimension::String)
return reader.metadata.dimension_size[index]
end

"""
close!(reader::Reader)
Close the reader.
"""
function close!(reader::Reader)
_quiver_close!(reader)
return nothing
Expand Down Expand Up @@ -148,6 +171,15 @@ function file_to_array(
return data, metadata
end

"""
file_to_df(
filename::String,
implementation::Type{I};
labels_to_read::Vector{String} = String[],
) where {I <: Implementation}
Reads a file and returns the data and metadata as a DataFrame.
"""
function file_to_df(
filename::String,
implementation::Type{I};
Expand Down
6 changes: 6 additions & 0 deletions test/test_read_write.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1115,6 +1115,9 @@ function read_file_to_array(impl)
for i in eachindex(data)
@test data[i] == data_read[i]
end

rm("$filename.$(Quiver.file_extension(impl))")
rm("$filename.toml")
end

function read_file_to_df(impl)
Expand Down Expand Up @@ -1164,6 +1167,9 @@ function read_file_to_df(impl)
@test DataFrames.metadata(df, "time_dimension") == "stage"
@test DataFrames.metadata(df, "dimensions") == ["stage", "scenario", "block"]
@test DataFrames.metadata(df, "labels") == ["agent_1", "agent_2", "agent_3"]

rm("$filename.$(Quiver.file_extension(impl))")
rm("$filename.toml")
end

function test_read_write_implementations()
Expand Down

0 comments on commit 3e2eab8

Please sign in to comment.