diff --git a/Project.toml b/Project.toml index ac21081..63f49ef 100644 --- a/Project.toml +++ b/Project.toml @@ -11,7 +11,7 @@ DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" GRIB = "b16dfd50-4035-11e9-28d4-9dfe17e6779b" [compat] -CommonDataModel = "^0.2.1, 0.3" +CommonDataModel = "0.3.4" DataStructures = "0.18" DiskArrays = "0.3" GRIB = "0.3, 0.4" diff --git a/src/GRIBDatasets.jl b/src/GRIBDatasets.jl index 419b3b9..662e990 100644 --- a/src/GRIBDatasets.jl +++ b/src/GRIBDatasets.jl @@ -5,7 +5,8 @@ using GRIB using DataStructures import DiskArrays as DA using CommonDataModel: AbstractDataset, AbstractVariable, show_dim, CFVariable -import CommonDataModel: path, name, dimnames, isopen, attribnames, attrib, dataset +import CommonDataModel: path, name, dimnames, isopen, attribnames, attrib, + dataset, maskingvalue import CommonDataModel as CDM const DEFAULT_EPOCH = DateTime(1970, 1, 1, 0, 0) diff --git a/src/cfvariables.jl b/src/cfvariables.jl index a92adc8..cfa584e 100644 --- a/src/cfvariables.jl +++ b/src/cfvariables.jl @@ -1,6 +1,6 @@ _get_dim(cfvar::CFVariable, dimname) = _get_dim(cfvar.var, dimname) -function cfvariable(ds, varname) +function cfvariable(ds, varname; maskingvalue = maskingvalue(ds)) v = Variable(ds, string(varname)) misval = missing_value(v) CDM.cfvariable( @@ -8,6 +8,7 @@ function cfvariable(ds, varname) _v = v, missing_value = isnothing(misval) ? eltype(v)[] : [misval], attrib = cflayer_attributes(v), + maskingvalue = maskingvalue, ) end diff --git a/src/dataset.jl b/src/dataset.jl index 431db8a..e50474c 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -8,19 +8,21 @@ It can be created with the path to the GRIB file: ds = GRIBDataset(example_file); ``` """ -struct GRIBDataset{T, N} <: AbstractDataset +struct GRIBDataset{T, N, Tmaskingvalue} <: AbstractDataset index::FileIndex{T} dims::NTuple{N, AbstractDim} attrib::Dict{String, Any} + maskingvalue::Tmaskingvalue end const Dataset = GRIBDataset -function GRIBDataset(index::FileIndex) - GRIBDataset(index, _alldims(index), dataset_attributes(index)) +function GRIBDataset(index::FileIndex; maskingvalue = missing) + GRIBDataset(index, _alldims(index), dataset_attributes(index), maskingvalue) end -GRIBDataset(filepath::AbstractString; filter_by_values = Dict()) = GRIBDataset(FileIndex(filepath; filter_by_values)) +GRIBDataset(filepath::AbstractString; filter_by_values = Dict(), kwargs...) = + GRIBDataset(FileIndex(filepath; filter_by_values); kwargs...) Base.keys(ds::Dataset) = getvars(ds) Base.haskey(ds::Dataset, key) = key in keys(ds) @@ -46,6 +48,7 @@ dimnames(ds::GRIBDataset) = keys(ds.dims) attribnames(ds::GRIBDataset) = keys(ds.attrib) attrib(ds::GRIBDataset, attribname::String) = ds.attrib[attribname] +maskingvalue(ds::GRIBDataset) = ds.maskingvalue # _dim_values(ds::GRIBDataset, dim::Dimension{Horizontal}) = _dim_values(ds.index, dim) diff --git a/test/dataset.jl b/test/dataset.jl index 2daea58..03b5498 100644 --- a/test/dataset.jl +++ b/test/dataset.jl @@ -12,6 +12,7 @@ using GRIBDatasets: CDM grib_path = joinpath(dir_testfiles, "era5-levels-members.grib") ds = GRIBDataset(grib_path) dsmis = GRIBDataset(joinpath(dir_testfiles, "fields_with_missing_values.grib")) + dsNaN = GRIBDataset(joinpath(dir_testfiles, "fields_with_missing_values.grib"),maskingvalue = NaN) index = ds.index varstring = "z" @@ -87,12 +88,25 @@ using GRIBDatasets: CDM @testset "cfvariable and missing" begin cfvar = cfvariable(ds, varstring) cfvarmis = cfvariable(dsmis, "t2m") + cfvarNaN = cfvariable(dsNaN, "t2m") A = cfvar[:,:,1,1,1] Amis = cfvarmis[:,:,1,1] + ANaN = cfvarNaN[:,:,1,1] # With CommonDataModel, we necessarily get a Union{Missing, Float64}, even if there's no missing. @test_broken eltype(A) == Float64 @test eltype(Amis) == Union{Missing, Float64} + @test eltype(ANaN) == Float64 + + # test the use of a different maskingvalue per dataset + @test eltype(dsmis["t2m"]) == Union{Missing,Float64} + @test eltype(dsNaN["t2m"]) == Float64 + @test ismissing(Amis[1,1,1]) + @test isnan(ANaN[1,1,1]) + + # test the use of a different maskingvalue per variable + A2NaN = cfvariable(dsmis,"t2m", maskingvalue = NaN) + @test isnan(A2NaN[1,1,1]) end @testset "cfvariable coordinate" begin @@ -181,4 +195,4 @@ end @time ds = GRIBDataset(testfile) end -end \ No newline at end of file +end