From 08fbc7169afef4fd6df47040658bd6307a46ece9 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 3 Jun 2022 12:31:02 +0200 Subject: [PATCH 01/37] Allow newest IntervalSets --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 4d8a7be..2bb0d39 100644 --- a/Project.toml +++ b/Project.toml @@ -24,7 +24,7 @@ ChainRulesCore = "1" ChainRulesTestUtils = "1" CovarianceEstimation = "0.2" FiniteDifferences = "0.12" -IntervalSets = "0.5.1, 0.6" +IntervalSets = "0.5.1, 0.6, 0.7" InvertedIndices = "1.0" LazyStack = "0.0.7, 0.0.8" NamedDims = "0.2.46" From 0ff9b3bb9f5fa93e651047f8aff82fa53a3cf743 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Fri, 3 Jun 2022 08:58:09 -0400 Subject: [PATCH 02/37] v0.2.4 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 2bb0d39..030d4ae 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.3" +version = "0.2.4" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" From a7bb1fcd08a7765f15dccff802f59cf9fa2c3cce Mon Sep 17 00:00:00 2001 From: Eric Davies Date: Wed, 8 Jun 2022 11:47:56 -0500 Subject: [PATCH 03/37] Fix test/speed.jl symlink --- test/speed.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/speed.jl b/test/speed.jl index 265dd3a..0d29edb 120000 --- a/test/speed.jl +++ b/test/speed.jl @@ -1 +1 @@ -docs/speed.jl \ No newline at end of file +../docs/speed.jl \ No newline at end of file From 648ba1e58d7ba97a038c42d6e3cb332251754ff9 Mon Sep 17 00:00:00 2001 From: Miha Zgubic Date: Wed, 6 Jul 2022 13:31:42 +0100 Subject: [PATCH 04/37] implement 1-arg cat/hcat/vcat (#115) * implement 1-arg cat/hcat/vcat * fix tests * simplify hcat --- src/functions.jl | 15 +++++++++++++++ test/_functions.jl | 14 ++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/functions.jl b/src/functions.jl index eea888f..e65a97b 100644 --- a/src/functions.jl +++ b/src/functions.jl @@ -173,6 +173,21 @@ for (T, S) in [ (:KeyedArray, :KeyedArray), end end +# single argument +Base.vcat(A::KeyedArray) = A +function Base.hcat(A::KeyedArray) + data = hcat(keyless(A)) + akeys = map(copy, (keys_or_axes(A, 1), keys_or_axes(A, 2))) + KeyedArray(data, akeys) +end +function Base.cat(A::KeyedArray; dims) + new_names = NamedDims.expand_dimnames(dimnames(A), dims) + numerical_dims = NamedDims.dim(new_names, dims) + data = cat(keyless(A); dims=dims) + new_keys = ntuple(d -> keys_or_axes(A, d), ndims(data)) + KeyedArray(data, map(copy, new_keys)) # , copy(A.meta)) +end + val_strip(dims::Val{d}) where {d} = d val_strip(dims) = dims key_vcat(a::AbstractVector, b::AbstractVector) = vcat(a,b) diff --git a/test/_functions.jl b/test/_functions.jl index 39ce520..40548b9 100644 --- a/test/_functions.jl +++ b/test/_functions.jl @@ -209,6 +209,20 @@ end @test_throws ArgumentError axiskeys(reduce(hcat, twomats')) @test_throws ArgumentError axiskeys(reduce(vcat, twomats)) + # single argument + @test dimnames(vcat(V)) == dimnames(cat(V; dims=1)) == (:r,) + @test axiskeys(vcat(V)) == axiskeys(cat(V; dims=1)) == (['a', 'b', 'c'],) + @test dimnames(hcat(V)) == dimnames(cat(V; dims=2)) == (:r, :_) + @test axiskeys(hcat(V)) == axiskeys(cat(V; dims=2)) == (['a', 'b', 'c'], Base.OneTo(1)) + + @test dimnames(vcat(M)) == dimnames(cat(M; dims=1)) == (:r, :c) + @test axiskeys(vcat(M)) == axiskeys(cat(M; dims=1)) == ('a':1:'c', 2:5) + + @test dimnames(hcat(M)) == dimnames(cat(M, dims=2)) == (:r, :c) + @test axiskeys(hcat(M)) == axiskeys(cat(M, dims=2)) == ('a':1:'c', 2:5) + + @test dimnames(cat(M; dims=3)) == (:r, :c, :_) + @test axiskeys(cat(M; dims=3)) == ('a':1:'c', 2:5, Base.OneTo(1)) end @testset "matmul" begin From 0d56ad91e17670c497b6f131f1f8f0d8fe0dba69 Mon Sep 17 00:00:00 2001 From: Miha Zgubic Date: Mon, 8 Aug 2022 15:11:49 +0100 Subject: [PATCH 05/37] Update Project.toml --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 030d4ae..fd02d03 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.4" +version = "0.2.5" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" @@ -27,7 +27,7 @@ FiniteDifferences = "0.12" IntervalSets = "0.5.1, 0.6, 0.7" InvertedIndices = "1.0" LazyStack = "0.0.7, 0.0.8" -NamedDims = "0.2.46" +NamedDims = "0.2.46, 0.3" OffsetArrays = "0.10, 0.11, 1.0" StatsBase = "0.32, 0.33" Tables = "0.2, 1" From f92b2b5f6545200eeb2a31ce06a5ac5b79331a24 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 10 Aug 2022 17:25:31 +0000 Subject: [PATCH 06/37] Performance tweak --- src/tables.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tables.jl b/src/tables.jl index 1d3f891..24bae0d 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -38,8 +38,12 @@ function Tables.columns(A::Union{KeyedArray, NdaKa}) L = hasnames(A) ? (dimnames(A)..., :value) : (ntuple(d -> Symbol(:dim_,d), ndims(A))..., :value) R = keys_or_axes(A) + + _get_col(R, ::Val{d}) where {d} = vec([rs[d] for rs in Iterators.product(R...)]) + G = ntuple(ndims(A)) do d - vec([rs[d] for rs in Iterators.product(R...)]) + _get_col(R, Val(d)) + # vec([rs[d] for rs in Iterators.product(R...)]) # _vec(rs[d] for rs in Iterators.product(R...)) end C = (G..., vec(parent(A))) From 33cbf1340d96ec34038dc3ccbf0c45adeb98d180 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 10 Aug 2022 17:44:48 +0000 Subject: [PATCH 07/37] Refine patch --- src/tables.jl | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 24bae0d..371dcf8 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -38,14 +38,27 @@ function Tables.columns(A::Union{KeyedArray, NdaKa}) L = hasnames(A) ? (dimnames(A)..., :value) : (ntuple(d -> Symbol(:dim_,d), ndims(A))..., :value) R = keys_or_axes(A) + # R_inds = ntuple(d -> eachindex(R[i]), length(R)) + R_inds = map(eachindex, R) + # Add function barrier and dispatch on column number for local type stability. _get_col(R, ::Val{d}) where {d} = vec([rs[d] for rs in Iterators.product(R...)]) - G = ntuple(ndims(A)) do d - _get_col(R, Val(d)) - # vec([rs[d] for rs in Iterators.product(R...)]) - # _vec(rs[d] for rs in Iterators.product(R...)) - end + inds = ntuple(identity, length(R)) + G = map( + (r, d) -> vec([r[rs[d]] for rs in Iterators.product(R_inds...)]), + R, inds, + ) + + # G = map(enumerate(R)) do (d, r) + # @show d, typeof(r) + # vec([r[rs[d]] for rs in Iterators.product(R_inds...)]) + # end + # G = ntuple(ndims(A)) do d + # # _get_col(R, Val(d)) + # vec([R[d][rs[d]] for rs in Iterators.product(R_inds...)]) + # # _vec(rs[d] for rs in Iterators.product(R...)) + # end C = (G..., vec(parent(A))) NamedTuple{L}(C) end From 5dbaf24f924cbd05d998c1ec257308e59af25c90 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 10 Aug 2022 17:53:00 +0000 Subject: [PATCH 08/37] Bump patch --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index fd02d03..626b579 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.5" +version = "0.2.6" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" From b7e59d5f37b2744db4c8231a9fcb75bfdddf2c43 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 10 Aug 2022 17:59:37 +0000 Subject: [PATCH 09/37] Tidy up --- src/tables.jl | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 371dcf8..99a1a5d 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -38,27 +38,11 @@ function Tables.columns(A::Union{KeyedArray, NdaKa}) L = hasnames(A) ? (dimnames(A)..., :value) : (ntuple(d -> Symbol(:dim_,d), ndims(A))..., :value) R = keys_or_axes(A) - # R_inds = ntuple(d -> eachindex(R[i]), length(R)) R_inds = map(eachindex, R) - - # Add function barrier and dispatch on column number for local type stability. - _get_col(R, ::Val{d}) where {d} = vec([rs[d] for rs in Iterators.product(R...)]) - - inds = ntuple(identity, length(R)) G = map( (r, d) -> vec([r[rs[d]] for rs in Iterators.product(R_inds...)]), - R, inds, + R, ntuple(identity, length(R)), ) - - # G = map(enumerate(R)) do (d, r) - # @show d, typeof(r) - # vec([r[rs[d]] for rs in Iterators.product(R_inds...)]) - # end - # G = ntuple(ndims(A)) do d - # # _get_col(R, Val(d)) - # vec([R[d][rs[d]] for rs in Iterators.product(R_inds...)]) - # # _vec(rs[d] for rs in Iterators.product(R...)) - # end C = (G..., vec(parent(A))) NamedTuple{L}(C) end From 3e61dec6f72218ddfc08ba89eb061e4844022f76 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 10 Aug 2022 18:21:04 +0000 Subject: [PATCH 10/37] Add comment explaining behaviour --- src/tables.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/tables.jl b/src/tables.jl index 99a1a5d..643148f 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -38,9 +38,18 @@ function Tables.columns(A::Union{KeyedArray, NdaKa}) L = hasnames(A) ? (dimnames(A)..., :value) : (ntuple(d -> Symbol(:dim_,d), ndims(A))..., :value) R = keys_or_axes(A) + + # R_inds comprises the indices for each of the keys. + R_inds = map(eachindex, R) + # indices is a tuple, the dth element of which is an index for the dth column of R. + # By using these indices, and mapping over the columns of R, the compiler seems to + # successfully infer the types in G, because it knows the element types of each column + # of R, so is presumably able to unroll the call to map. + # The previous implementation called `Iterators.product` on `R` and pulled out + # the dth element of `indices`, whose type it could not infer. G = map( - (r, d) -> vec([r[rs[d]] for rs in Iterators.product(R_inds...)]), + (r, d) -> vec([r[indices[d]] for indices in Iterators.product(R_inds...)]), R, ntuple(identity, length(R)), ) C = (G..., vec(parent(A))) From 577622aa364aef3443969a588252f41cfa631324 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 10 Aug 2022 18:35:11 +0000 Subject: [PATCH 11/37] Factor out new code --- src/tables.jl | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 643148f..03d5c0f 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -37,23 +37,24 @@ Tables.columnaccess(::Type{<:KeyedArray{T,N,AT}}) where {T,N,AT} = function Tables.columns(A::Union{KeyedArray, NdaKa}) L = hasnames(A) ? (dimnames(A)..., :value) : (ntuple(d -> Symbol(:dim_,d), ndims(A))..., :value) - R = keys_or_axes(A) + G = _get_keys_columns(A) + C = (G..., vec(parent(A))) + NamedTuple{L}(C) +end - # R_inds comprises the indices for each of the keys. - +# indices is a tuple, the dth element of which is an index for the dth column of R. +# By using these indices, and mapping over the columns of R, the compiler seems to +# successfully infer the types in G, because it knows the element types of each column +# of R, so is presumably able to unroll the call to map. +# The previous implementation called `Iterators.product` on `R` and pulled out +# the dth element of `indices`, whose type it could not infer. +function _get_keys_columns(A) + R = keys_or_axes(A) R_inds = map(eachindex, R) - # indices is a tuple, the dth element of which is an index for the dth column of R. - # By using these indices, and mapping over the columns of R, the compiler seems to - # successfully infer the types in G, because it knows the element types of each column - # of R, so is presumably able to unroll the call to map. - # The previous implementation called `Iterators.product` on `R` and pulled out - # the dth element of `indices`, whose type it could not infer. - G = map( + return map( (r, d) -> vec([r[indices[d]] for indices in Iterators.product(R_inds...)]), R, ntuple(identity, length(R)), ) - C = (G..., vec(parent(A))) - NamedTuple{L}(C) end function Tables.Schema(nt::NamedTuple) # 🏴‍☠️ From d2364315d460123c98aee2c87dae2ae5c8bda093 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Thu, 11 Aug 2022 08:57:06 +0000 Subject: [PATCH 12/37] Refactor + add regression test --- src/tables.jl | 5 ++--- test/_packages.jl | 9 +++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 03d5c0f..91694cc 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -37,7 +37,7 @@ Tables.columnaccess(::Type{<:KeyedArray{T,N,AT}}) where {T,N,AT} = function Tables.columns(A::Union{KeyedArray, NdaKa}) L = hasnames(A) ? (dimnames(A)..., :value) : (ntuple(d -> Symbol(:dim_,d), ndims(A))..., :value) - G = _get_keys_columns(A) + G = _get_keys_columns(keys_or_axes(A)) C = (G..., vec(parent(A))) NamedTuple{L}(C) end @@ -48,8 +48,7 @@ end # of R, so is presumably able to unroll the call to map. # The previous implementation called `Iterators.product` on `R` and pulled out # the dth element of `indices`, whose type it could not infer. -function _get_keys_columns(A) - R = keys_or_axes(A) +function _get_keys_columns(R) R_inds = map(eachindex, R) return map( (r, d) -> vec([r[indices[d]] for indices in Iterators.product(R_inds...)]), diff --git a/test/_packages.jl b/test/_packages.jl index b5812a9..1f32f9e 100644 --- a/test/_packages.jl +++ b/test/_packages.jl @@ -49,6 +49,15 @@ end @test keys(first(Tables.rows(N))) == (:a, :b, :value) @test Tables.columns(N).a == [11, 12, 11, 12, 11, 12] + + # Regression test: https://github.com/mcabbott/AxisKeys.jl/pull/123 + @testset "Tables.columns(::AxisKeys) type stability" begin + a = randn(5) + b = randn(Float32, 4) + @inferred AxisKeys._get_keys_columns((a, b)) + X = wrapdims(randn(5, 4); a, b) + @inferred Tables.columns(X) + end end @testset "sink" begin A = KeyedArray(rand(24, 11, 3); time = 0:23, loc = -5:5, id = ["a", "b", "c"]) From 0d36405e0002a880938f1fcf7e1f9aad22fadfac Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Thu, 11 Aug 2022 09:03:47 +0000 Subject: [PATCH 13/37] Do-block syntax --- src/tables.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 91694cc..6e60d84 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -50,10 +50,9 @@ end # the dth element of `indices`, whose type it could not infer. function _get_keys_columns(R) R_inds = map(eachindex, R) - return map( - (r, d) -> vec([r[indices[d]] for indices in Iterators.product(R_inds...)]), - R, ntuple(identity, length(R)), - ) + return map(R, ntuple(identity, length(R))) do r, d + vec([r[indices[d]] for indices in Iterators.product(R_inds...)]) + end end function Tables.Schema(nt::NamedTuple) # 🏴‍☠️ From ab13a9309fc73de67616671d3c4e156a583e746c Mon Sep 17 00:00:00 2001 From: Andrew Radcliffe <96091198+andrewjradcliffe@users.noreply.github.com> Date: Thu, 11 Aug 2022 22:36:19 -0700 Subject: [PATCH 14/37] Replace `_maybetail` with `safe_tail` (#124) This was motivated by the disappearance of `_maybetail` from `Base`, which causes `AxisKeys` to not be compatible with v1.9. As it happens, `safe_tail` is identical to `_maybetail`, thus, it would seem that the latter was eliminated due to redundancy. Inspection of the v1.6 branch of JuliaLang confirms that `safe_tail` exists in its same form, thus, given the `[compat]` for this `AxisKeys`, this seems like a harmless change which enables compatibility with `master`. --- src/selectors.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/selectors.jl b/src/selectors.jl index d4eb223..3963d8d 100644 --- a/src/selectors.jl +++ b/src/selectors.jl @@ -208,8 +208,8 @@ end @inline Base.to_indices(A::Union{KeyedArray,NdaKa}, ax, inds::Tuple{Function, Vararg}) = select_to_indices(A, ax, inds) -using Base: to_indices, tail, _maybetail, uncolon +using Base: to_indices, tail, safe_tail, uncolon @inline Base.to_indices(A::Union{KeyedArray,NdaKa}, inds, I::Tuple{Colon, Vararg{Any}}) = - (uncolon(inds, I), to_indices(A, _maybetail(inds), tail(I))...) + (uncolon(inds, I), to_indices(A, safe_tail(inds), tail(I))...) From ea2c7fd4cf09179cb9f381457e4ea1932ce1d02f Mon Sep 17 00:00:00 2001 From: Miha Zgubic Date: Fri, 12 Aug 2022 09:56:22 +0100 Subject: [PATCH 15/37] Update Project.toml --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 626b579..10a8695 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.6" +version = "0.2.7" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" @@ -27,7 +27,7 @@ FiniteDifferences = "0.12" IntervalSets = "0.5.1, 0.6, 0.7" InvertedIndices = "1.0" LazyStack = "0.0.7, 0.0.8" -NamedDims = "0.2.46, 0.3" +NamedDims = "0.2.46, 0.3, 1" OffsetArrays = "0.10, 0.11, 1.0" StatsBase = "0.32, 0.33" Tables = "0.2, 1" From da92a7fa5b42d2de0738ce14038aaca2259d2963 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 31 Aug 2022 21:55:40 +0100 Subject: [PATCH 16/37] New implementation --- src/tables.jl | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 6e60d84..1747f77 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -134,29 +134,57 @@ corresponding to the keys in `A` and implements `Tables.rows`. If the keys in `A uniquely identify rows in the `table` then an `ArgumentError` is throw. If `force` is true then the duplicate (non-unique) entries will be overwritten. """ +# function populate!(A, table, value::Symbol; force=false) +# # Use a BitArray mask to detect duplicates and error instead of overwriting. +# mask = force ? falses() : falses(size(A)) + +# for r in Tables.rows(table) +# vals = Tuple(Tables.getcolumn(r, c) for c in dimnames(A)) +# inds = map(findindex, vals, axiskeys(A)) + +# # Handle duplicate error checking if applicable +# if !force +# # Error if mask already set. +# mask[inds...] && throw(ArgumentError("Key $vals is not unique")) +# # Set mask, marking that we've set this index +# setindex!(mask, true, inds...) +# end + +# # Insert our value into the data array +# setindex!(A, Tables.getcolumn(r, value), inds...) +# end + +# return A +# end + function populate!(A, table, value::Symbol; force=false) # Use a BitArray mask to detect duplicates and error instead of overwriting. mask = force ? falses() : falses(size(A)) - for r in Tables.rows(table) - vals = Tuple(Tables.getcolumn(r, c) for c in dimnames(A)) - inds = map(findindex, vals, axiskeys(A)) + cols = Tables.columns(table) + value_column = Tables.getcolumn(cols, value) + axis_key_columns = Tuple(Tables.getcolumn(cols, c) for c in dimnames(A)) + return populate_function_barrier!(A, value_column, axis_key_columns, mask, force) +end + +function populate_function_barrier!(A, value_column, axis_key_columns, mask, force) + for (val, keys...) in zip(value_column, axis_key_columns...) + inds = map(AxisKeys.findindex, keys, axiskeys(A)) # Handle duplicate error checking if applicable if !force # Error if mask already set. - mask[inds...] && throw(ArgumentError("Key $vals is not unique")) + mask[inds...] && throw(ArgumentError("Key $keys is not unique")) # Set mask, marking that we've set this index setindex!(mask, true, inds...) end - # Insert our value into the data array - setindex!(A, Tables.getcolumn(r, value), inds...) + setindex!(A, val, inds...) end - return A end + """ wrapdims(table, value, names...; default=undef, sort=false, force=false) From f39a000d6fb90c54d518bd6f990500c6d8d9bc7c Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 31 Aug 2022 23:30:25 +0100 Subject: [PATCH 17/37] Remove redundant comment Co-authored-by: Eric Davies --- src/tables.jl | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 1747f77..549f8a8 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -134,29 +134,6 @@ corresponding to the keys in `A` and implements `Tables.rows`. If the keys in `A uniquely identify rows in the `table` then an `ArgumentError` is throw. If `force` is true then the duplicate (non-unique) entries will be overwritten. """ -# function populate!(A, table, value::Symbol; force=false) -# # Use a BitArray mask to detect duplicates and error instead of overwriting. -# mask = force ? falses() : falses(size(A)) - -# for r in Tables.rows(table) -# vals = Tuple(Tables.getcolumn(r, c) for c in dimnames(A)) -# inds = map(findindex, vals, axiskeys(A)) - -# # Handle duplicate error checking if applicable -# if !force -# # Error if mask already set. -# mask[inds...] && throw(ArgumentError("Key $vals is not unique")) -# # Set mask, marking that we've set this index -# setindex!(mask, true, inds...) -# end - -# # Insert our value into the data array -# setindex!(A, Tables.getcolumn(r, value), inds...) -# end - -# return A -# end - function populate!(A, table, value::Symbol; force=false) # Use a BitArray mask to detect duplicates and error instead of overwriting. mask = force ? falses() : falses(size(A)) From b14fc589c13c31828a3ff8d196a7eaa8ccb9ae88 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 31 Aug 2022 23:43:10 +0100 Subject: [PATCH 18/37] Add comment --- src/tables.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tables.jl b/src/tables.jl index 549f8a8..675d51e 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -144,6 +144,10 @@ function populate!(A, table, value::Symbol; force=false) return populate_function_barrier!(A, value_column, axis_key_columns, mask, force) end +# eltypes of value and axis_key_columns aren't inferable in `populate!` if the `table` +# doesn't have typed columns, as is the case for DataFrames. By passing them into +# `populate_function_barrier!` once they've been pulled out of a DataFrame ensures +# inference is possible for the loop. function populate_function_barrier!(A, value_column, axis_key_columns, mask, force) for (val, keys...) in zip(value_column, axis_key_columns...) inds = map(AxisKeys.findindex, keys, axiskeys(A)) From 46524824e9165058bdc2ca75b1c5bdef6cafe76f Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 31 Aug 2022 23:43:40 +0100 Subject: [PATCH 19/37] Bump patch --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 10a8695..9045fb4 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.7" +version = "0.2.8" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" From 44f10623f3b2ad6ba64509385f43089de6b4ee98 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 31 Aug 2022 23:44:06 +0100 Subject: [PATCH 20/37] Revert addition of line --- src/tables.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tables.jl b/src/tables.jl index 675d51e..526d833 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -165,7 +165,6 @@ function populate_function_barrier!(A, value_column, axis_key_columns, mask, for return A end - """ wrapdims(table, value, names...; default=undef, sort=false, force=false) From 2d8fd20fd3539d036fbb08b8ef78f1701aa325f7 Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Wed, 31 Aug 2022 23:59:26 +0100 Subject: [PATCH 21/37] Add lightweight performance test --- Project.toml | 4 +++- test/_packages.jl | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 9045fb4..7b74ebb 100644 --- a/Project.toml +++ b/Project.toml @@ -23,6 +23,7 @@ BenchmarkTools = "0.5, 1.0" ChainRulesCore = "1" ChainRulesTestUtils = "1" CovarianceEstimation = "0.2" +DataFrames = "1" FiniteDifferences = "0.12" IntervalSets = "0.5.1, 0.6, 0.7" InvertedIndices = "1.0" @@ -36,6 +37,7 @@ julia = "1.6" [extras] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" @@ -45,4 +47,4 @@ UniqueVectors = "2fbcfb34-fd0c-5fbb-b5d7-e826d8f5b0a9" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" [targets] -test = ["BenchmarkTools", "ChainRulesTestUtils", "Dates", "FiniteDifferences", "FFTW", "NamedArrays", "Test", "UniqueVectors", "Unitful"] +test = ["BenchmarkTools", "ChainRulesTestUtils", "DataFrames", "Dates", "FiniteDifferences", "FFTW", "NamedArrays", "Test", "UniqueVectors", "Unitful"] diff --git a/test/_packages.jl b/test/_packages.jl index 1f32f9e..3f188e0 100644 --- a/test/_packages.jl +++ b/test/_packages.jl @@ -1,5 +1,10 @@ using Test, AxisKeys +function count_allocs(f, args...) + stats = @timed f(args...) + return Base.gc_alloc_count(stats.gcstats) +end + @testset "offset" begin using OffsetArrays @@ -38,6 +43,14 @@ end @test dimnames(k) == (:aa,) end end +@testset "DataFrames" begin + using DataFrames + + X = KeyedArray(randn(1000, 1500), a=1:1000, b=1:1500) + df = DataFrame(X) + wrapdims(df, :value, :a, :b) # compile + @test count_allocs(wrapdims, df, :value, :a, :b) < 1_000 +end @testset "tables" begin using Tables From a91c7f834a1e6e9c2330d085d304adbacf353c9c Mon Sep 17 00:00:00 2001 From: Will Tebbutt Date: Thu, 1 Sep 2022 00:07:50 +0100 Subject: [PATCH 22/37] Limit imports --- test/_packages.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/_packages.jl b/test/_packages.jl index 3f188e0..50facda 100644 --- a/test/_packages.jl +++ b/test/_packages.jl @@ -44,7 +44,7 @@ end end end @testset "DataFrames" begin - using DataFrames + using DataFrames: DataFrame X = KeyedArray(randn(1000, 1500), a=1:1000, b=1:1500) df = DataFrame(X) From 596c163e98113cf48e653d35423223e2170eec38 Mon Sep 17 00:00:00 2001 From: rofinn Date: Sat, 3 Sep 2022 20:05:33 -0600 Subject: [PATCH 23/37] Fix #128 broadcast ambiguities. --- Project.toml | 2 +- src/broadcast.jl | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index 7b74ebb..6872b6f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.8" +version = "0.2.9" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" diff --git a/src/broadcast.jl b/src/broadcast.jl index 972b9ad..f33e253 100644 --- a/src/broadcast.jl +++ b/src/broadcast.jl @@ -21,14 +21,25 @@ Base.BroadcastStyle(::Type{<:KeyedArray{T,N,AT}}) where {T,N,AT} = Base.BroadcastStyle(::KeyedStyle{A}, ::KeyedStyle{B}) where {A, B} = KeyedStyle(A(), B()) Base.BroadcastStyle(::KeyedStyle{A}, b::B) where {A, B} = KeyedStyle(A(), b) Base.BroadcastStyle(a::A, ::KeyedStyle{B}) where {A, B} = KeyedStyle(a, B()) -Base.BroadcastStyle(::KeyedStyle{A}, b::DefaultArrayStyle) where {A} = KeyedStyle(A(), b) -Base.BroadcastStyle(a::AbstractArrayStyle{M}, ::KeyedStyle{B}) where {B,M} = KeyedStyle(a, B()) using NamedDims: NamedDimsStyle # this resolves in favour of KeyedArray(NamedDimsArray()) Base.BroadcastStyle(a::NamedDimsStyle, ::KeyedStyle{B}) where {B} = KeyedStyle(a, B()) Base.BroadcastStyle(::KeyedStyle{A}, b::NamedDimsStyle) where {A} = KeyedStyle(A(), b) +# Resolve ambiguities +# for all these cases, we define that we win to be the outer style regardless of order +for B in ( + :BroadcastStyle, :DefaultArrayStyle, :AbstractArrayStyle, :(Broadcast.Style{Tuple}), +) + @eval function Base.BroadcastStyle(::KeyedStyle{A}, b::$B) where A + return KeyedStyle(A(), b) + end + @eval function Base.BroadcastStyle(b::$B, ::KeyedStyle{A}) where A + return KeyedStyle(b, A()) + end +end + function unwrap_broadcasted(bc::Broadcasted{KeyedStyle{S}}) where {S} inner_args = map(unwrap_broadcasted, bc.args) Broadcasted{S}(bc.f, inner_args) From 46db5c9ed65327d160942f1336286ac1eaf30328 Mon Sep 17 00:00:00 2001 From: rofinn Date: Mon, 5 Sep 2022 22:47:47 +0100 Subject: [PATCH 24/37] Add a simple test for this specific issue, and a fix to unwrap_broadcasted. --- src/broadcast.jl | 2 +- test/_basic.jl | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/broadcast.jl b/src/broadcast.jl index f33e253..e260483 100644 --- a/src/broadcast.jl +++ b/src/broadcast.jl @@ -42,7 +42,7 @@ end function unwrap_broadcasted(bc::Broadcasted{KeyedStyle{S}}) where {S} inner_args = map(unwrap_broadcasted, bc.args) - Broadcasted{S}(bc.f, inner_args) + Broadcasted{S}(bc.f, inner_args, axes(bc)) end unwrap_broadcasted(x) = x unwrap_broadcasted(x::KeyedArray) = parent(x) diff --git a/test/_basic.jl b/test/_basic.jl index 822375c..0b8e1b4 100644 --- a/test/_basic.jl +++ b/test/_basic.jl @@ -259,6 +259,17 @@ end @test_throws Exception unify_keys((1:2,), ([3,4],)) end + @testset "https://github.com/mcabbott/AxisKeys.jl/issues/128" begin + using LinearAlgebra + + # Really simple test that broadcasting with linalg array types works. + v = rand(10) + σ = wrapdims(v; time=-4:5) + L = LowerTriangular(reshape(1.0:1.0:100, (10, 10))) + σ .* L + v .* L + @test σ .* L ≈ v .* L + end end @testset "bitarray" begin From a96b5b54ce02c471cde7c409ebaa96d19776a99e Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 25 Oct 2022 10:33:24 -0700 Subject: [PATCH 25/37] Support rekeying a KeyedArray (#63) * Support various kinds of rekeying/renaming keys. * Simplify revise rekey signatures and try to keep things tuple-ish. * Dropped method to rekey and rename at the same time * Bump patch release since we're pre-1.0 --- Project.toml | 2 +- src/AxisKeys.jl | 2 +- src/names.jl | 24 ++++++++++++++++++++++++ test/_basic.jl | 10 ++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 6872b6f..3015728 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.9" +version = "0.2.10" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" diff --git a/src/AxisKeys.jl b/src/AxisKeys.jl index e90da5a..adeabcb 100644 --- a/src/AxisKeys.jl +++ b/src/AxisKeys.jl @@ -6,7 +6,7 @@ export KeyedArray, axiskeys include("lookup.jl") include("names.jl") -export named_axiskeys +export named_axiskeys, rekey export NamedDimsArray, dimnames, rename # Reexport key NamedDimsArrays things include("wrap.jl") diff --git a/src/names.jl b/src/names.jl index e533409..4bf4204 100644 --- a/src/names.jl +++ b/src/names.jl @@ -173,3 +173,27 @@ function named_axiskeys(A::AbstractArray) NT = NamedTuple{dimnames(A)} NT(axiskeys(A)) end + + +""" + rekey(A, (1:10, [:a, :b])) + rekey(A, 2 => [:a, :b]) + rekey(A, :y => [:a, :b]) + +Rekey a KeyedArray via `Tuple`s or `Pair`s, `dim => newkey`. If `A` also has named +dimensions then you can also pass `dimname => newkey`. +""" +rekey(A::Union{KeyedArray, NdaKa}, k2::Tuple) = KeyedArray(keyless(A), k2) +function rekey(A::Union{KeyedArray, NdaKa}, k2::Pair{<:Integer, <:AbstractVector}...) + dims, vals = first.(k2), last.(k2) + new_key = ntuple(ndims(A)) do d + idx = findfirst(==(d), dims) + idx === nothing ? axiskeys(A, d) : vals[idx] + end + return rekey(A, new_key) +end + +function rekey(A::Union{KaNda, NdaKa}, k2::Pair{Symbol, <:AbstractVector}...) + pairs = map(p -> NamedDims.dim(A, p[1]) => p[2], k2) + return rekey(A, pairs...) +end diff --git a/test/_basic.jl b/test/_basic.jl index 0b8e1b4..de90d1d 100644 --- a/test/_basic.jl +++ b/test/_basic.jl @@ -170,6 +170,16 @@ end @test_throws Exception N(obs=55) # ideally ArgumentError @test_throws Exception N(obs='z') # ideally BoundsError + + new_obs = [:x, :y, :z] + new_iter = 1:4 + + # Rekey with Tuple + @test axiskeys(rekey(N, ([:x, :y, :z], 1:4))) == ([:x, :y, :z], 1:4) + # Rekey with dim => axiskey pair + @test axiskeys(rekey(N, 2 => 1:4)) == (['a', 'b', 'c'], 1:4) + # Rekey with dimname => axiskey pair + @test axiskeys(rekey(N, :iter => 1:4)) == (['a', 'b', 'c'], 1:4) end @testset "named_axiskeys" begin From b9f3e37f237254a6f302ecc135cca75a6c5ecddb Mon Sep 17 00:00:00 2001 From: Matthew Priddin Date: Fri, 3 Feb 2023 17:31:57 +0000 Subject: [PATCH 26/37] Define `empty!(::KeyedVector)` --- src/functions.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/functions.jl b/src/functions.jl index e65a97b..3d152a7 100644 --- a/src/functions.jl +++ b/src/functions.jl @@ -441,4 +441,9 @@ function Base.filter!(f, a::KeyedVector) deleteat!(a, j:lastindex(a)) return a end - + +function Base.empty!(v::KeyedVector) + empty!(axiskeys(v, 1)) + empty!(v.data) + return v +end From 75f5f2d69fdd676f602e2724d1cb37aa4699b697 Mon Sep 17 00:00:00 2001 From: Matthew Priddin Date: Fri, 3 Feb 2023 17:32:05 +0000 Subject: [PATCH 27/37] Add tests --- test/_functions.jl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/_functions.jl b/test/_functions.jl index 40548b9..83410bc 100644 --- a/test/_functions.jl +++ b/test/_functions.jl @@ -148,7 +148,7 @@ end @test axiskeys(filter(isodd, V2),1) isa Vector{Int} @test dimnames(filter(isodd, V2)) == (:v,) - + V4 = wrapdims(rand(1:99, 10), collect(2:11)) V4c = copy(V4) @test filter!(isodd, V4c) === V4c == filter(isodd, V4) @@ -288,6 +288,17 @@ end @test ka == KeyedArray([4, 5, 6.0], a=1:3) end +@testset "empty!" begin + kv = wrapdims([1, 2, 3, 4, 5, 6.0], a=[:a, :b, :c, :d, :e, :f]) + @test kv == empty!(kv) + @test isempty(kv) + + # make sure array is not in an invalid state if the emtpy for indices fails + ka = wrapdims([4, 5, 6.0], a=1:3) + @test_throws MethodError empty!(ka) + @test ka == KeyedArray([4, 5, 6.0], a=1:3) +end + @testset "equality" begin data = parent(parent(M)) From e65d5feaf8f9de50d1177c76acff8d7162e260a6 Mon Sep 17 00:00:00 2001 From: Matthew Priddin Date: Fri, 3 Feb 2023 17:32:15 +0000 Subject: [PATCH 28/37] Bump version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 3015728..f00e2e5 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.10" +version = "0.2.11" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" From 0a81300582d3e2808c857e9c58eeadb6465920f5 Mon Sep 17 00:00:00 2001 From: Oscar Smith Date: Sat, 4 Feb 2023 23:26:17 -0500 Subject: [PATCH 29/37] remove deprecated `VarArg{<:...}` (#137) * remove deprecated `VarArg{<:...}` * Update statsbase.jl --- src/stack.jl | 4 ++-- src/statsbase.jl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/stack.jl b/src/stack.jl index 2ac7e97..0d035dc 100644 --- a/src/stack.jl +++ b/src/stack.jl @@ -14,11 +14,11 @@ function LazyStack.rewrap_like(A, a::NamedTuple) end # tuple of arrays -function LazyStack.stack(x::Tuple{Vararg{<:KeyedArray}}) +function LazyStack.stack(x::Tuple{Vararg{KeyedArray}}) KeyedArray(LazyStack.stack(map(parent, x)), stack_keys(x)) end -stack_keys(xs::Tuple{Vararg{<:KeyedArray}}) = +stack_keys(xs::Tuple{Vararg{KeyedArray}}) = (keys_or_axes(first(xs))..., Base.OneTo(length(xs))) # array of arrays: first strip off outer containers... diff --git a/src/statsbase.jl b/src/statsbase.jl index 0e2884a..d045e71 100644 --- a/src/statsbase.jl +++ b/src/statsbase.jl @@ -48,7 +48,7 @@ end for fun in (:std, :var, :cov) full_name = Symbol("mean_and_$fun") - @eval StatsBase.$full_name(A::KeyedMatrix, wv::Vararg{<:AbstractWeights}; dims=:, corrected::Bool=true, kwargs...) = + @eval StatsBase.$full_name(A::KeyedMatrix, wv::Vararg{AbstractWeights}; dims=:, corrected::Bool=true, kwargs...) = ( mean(A, wv...; dims=dims, kwargs...), $fun(A, wv...; dims=dims, corrected=corrected, kwargs...) From 79295189e73bc714cce7404bbb66efab7d638d48 Mon Sep 17 00:00:00 2001 From: Frames Catherine White Date: Sun, 5 Feb 2023 04:37:06 +0000 Subject: [PATCH 30/37] Fix docs that were outdated by previous PR (#127) --- src/tables.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tables.jl b/src/tables.jl index 526d833..81c1648 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -130,7 +130,7 @@ end Populate `A` with the contents of the `value` column in a provided `table`, matching the [Tables.jl](https://github.com/JuliaData/Tables.jl) API. The `table` must contain columns -corresponding to the keys in `A` and implements `Tables.rows`. If the keys in `A` do not +corresponding to the keys in `A` and implement `Tables.columns`. If the keys in `A` do not uniquely identify rows in the `table` then an `ArgumentError` is throw. If `force` is true then the duplicate (non-unique) entries will be overwritten. """ From 8d73d2a48b84dd762e79b8ec677e996ff3ed5223 Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Sun, 12 Feb 2023 03:25:51 +0300 Subject: [PATCH 31/37] fixes and updates for Julia 1.9 (#130) * update uncolon * eachslice with 1.9 semantics * add stack * add TODO * fix tests on 1.9 * properly qualify stack * remove accidental piracy * Update src/functions.jl --------- Co-authored-by: Michael Abbott <32575566+mcabbott@users.noreply.github.com> --- src/functions.jl | 21 ++++++++++++++++++++- src/selectors.jl | 10 ++++++++-- src/stack.jl | 8 ++++---- test/_basic.jl | 2 +- test/_functions.jl | 11 +++++++++++ test/_packages.jl | 20 ++++++++++---------- test/runtests.jl | 2 +- 7 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/functions.jl b/src/functions.jl index 3d152a7..f1758f8 100644 --- a/src/functions.jl +++ b/src/functions.jl @@ -94,7 +94,13 @@ function Base.permutedims(A::KeyedArray, perm) KeyedArray(data, new_keys)#, copy(A.meta)) end -if VERSION >= v"1.1" +@static if VERSION > v"1.9-DEV" + function Base.eachslice(A::KeyedArray; dims) + dims_ix = AxisKeys.dim(A, dims) |> Tuple + data = @invoke eachslice(A::AbstractArray; dims=dims_ix) + return KeyedArray(NamedDimsArray(data, map(d -> dimnames(A, d), dims_ix)), map(d -> axiskeys(A, d), dims_ix)) + end +elseif VERSION >= v"1.1" # This copies the implementation from Base, except with numerical_dims: @inline function Base.eachslice(A::KeyedArray; dims) numerical_dims = NamedDims.dim(A, dims) @@ -107,6 +113,19 @@ if VERSION >= v"1.1" end end +@static if VERSION > v"1.9-DEV" + # TODO: this will ERROR if given dims, instead of falling back to Base + # TODO: ideally it would dispatch on the element type, for e.g. a generator of KeyedArrays + function Base.stack(A::KeyedArray; dims::Colon=:) + data = @invoke stack(A::AbstractArray; dims) + if !allequal(named_axiskeys(a) for a in A) + throw(DimensionMismatch("stack expects uniform axiskeys for all arrays")) + end + akeys = (; named_axiskeys(first(A))..., named_axiskeys(A)...) + KeyedArray(data; akeys...) + end +end + function Base.mapslices(f, A::KeyedArray; dims) numerical_dims = NamedDims.dim(A, dims) data = mapslices(f, parent(A); dims=numerical_dims) diff --git a/src/selectors.jl b/src/selectors.jl index 3963d8d..62f5917 100644 --- a/src/selectors.jl +++ b/src/selectors.jl @@ -208,8 +208,14 @@ end @inline Base.to_indices(A::Union{KeyedArray,NdaKa}, ax, inds::Tuple{Function, Vararg}) = select_to_indices(A, ax, inds) -using Base: to_indices, tail, safe_tail, uncolon +using Base: to_indices, tail, safe_tail + +if VERSION > v"1.9-DEV" + const uncolon = Base.uncolon +else + uncolon(inds) = Base.uncolon(inds, (:,)) +end @inline Base.to_indices(A::Union{KeyedArray,NdaKa}, inds, I::Tuple{Colon, Vararg{Any}}) = - (uncolon(inds, I), to_indices(A, safe_tail(inds), tail(I))...) + (uncolon(inds), to_indices(A, safe_tail(inds), tail(I))...) diff --git a/src/stack.jl b/src/stack.jl index 0d035dc..1849591 100644 --- a/src/stack.jl +++ b/src/stack.jl @@ -1,5 +1,5 @@ -using LazyStack +import LazyStack # for stack_iter LazyStack.no_wraps(a::KeyedArray) = LazyStack.no_wraps(NamedDims.unname(parent(a))) @@ -23,14 +23,14 @@ stack_keys(xs::Tuple{Vararg{KeyedArray}}) = # array of arrays: first strip off outer containers... function LazyStack.stack(xs::KeyedArray{<:AbstractArray}) - KeyedArray(stack(parent(xs)), stack_keys(xs)) + KeyedArray(LazyStack.stack(parent(xs)), stack_keys(xs)) end function LazyStack.stack(xs::KeyedArray{<:AbstractArray,N,<:NamedDimsArray{L}}) where {L,N} - data = stack(parent(parent(xs))) + data = LazyStack.stack(parent(parent(xs))) KeyedArray(LazyStack.ensure_named(data, LazyStack.getnames(xs)), stack_keys(xs)) end function LazyStack.stack(xs::NamedDimsArray{L,<:AbstractArray,N,<:KeyedArray}) where {L,N} - data = stack(parent(parent(xs))) + data = LazyStack.stack(parent(parent(xs))) LazyStack.ensure_named(KeyedArray(data, stack_keys(xs)), LazyStack.getnames(xs)) end diff --git a/test/_basic.jl b/test/_basic.jl index de90d1d..59db1ca 100644 --- a/test/_basic.jl +++ b/test/_basic.jl @@ -151,7 +151,7 @@ end N2 = NamedDimsArray(KeyedArray(data, axiskeys(N1)), dimnames(N1)) N3 = KeyedArray(NamedDimsArray(data, dimnames(N1)), axiskeys(N1)) - @testset "with $(typeof(N).name) outside" for N in [N2, N3] + @testset "with $(typeof(N).name) outside" for N in (N2, N3) @test axiskeys(N) == (['a', 'b', 'c'], 10:10:40) @test axiskeys(N, :iter) == 10:10:40 @test dimnames(N) == (:obs, :iter) diff --git a/test/_functions.jl b/test/_functions.jl index 83410bc..189bf1e 100644 --- a/test/_functions.jl +++ b/test/_functions.jl @@ -46,6 +46,17 @@ A3 = wrapdims(rand(Int8, 3,4,2), r='a':'c', c=2:5, p=[10.0, 20.0]) if VERSION >= v"1.1" @test axiskeys(first(eachslice(M, dims=:r))) === (2:5,) end + if VERSION >= v"1.9-DEV" + arr = KeyedArray([ + KeyedArray([1, 2], a=[:x, :y]), + KeyedArray([3, 4], a=[:x, :y]), + KeyedArray([5, 6], a=[:x, :y]), + ], b=10:12) + + sk = stack(arr)::KeyedArray + @test sk == [1 3 5; 2 4 6] + @test named_axiskeys(sk) == (a=[:x, :y], b=10:12) + end @test axiskeys(selectdim(M, :r, [true, false, true])) == (['a', 'c'], 2:5) diff --git a/test/_packages.jl b/test/_packages.jl index 50facda..35bfa08 100644 --- a/test/_packages.jl +++ b/test/_packages.jl @@ -129,25 +129,25 @@ end end end @testset "stack" begin - using LazyStack + using LazyStack: stack as lstack rin = [wrapdims(1:3, a='a':'c') for i=1:4] - @test axiskeys(stack(rin), :a) == 'a':'c' - @test axiskeys(stack(:b, rin...), :a) == 'a':'c' # tuple - @test axiskeys(stack(z for z in rin), :a) == 'a':'c' # generator + @test axiskeys(lstack(rin), :a) == 'a':'c' + @test axiskeys(lstack(:b, rin...), :a) == 'a':'c' # tuple + @test axiskeys(lstack(z for z in rin), :a) == 'a':'c' # generator rout = wrapdims([[1,2], [3,4]], b=10:11) - @test axiskeys(stack(rout), :b) == 10:11 + @test axiskeys(lstack(rout), :b) == 10:11 rboth = wrapdims(rin, b=10:13) - @test axiskeys(stack(rboth), :a) == 'a':'c' - @test axiskeys(stack(rboth), :b) == 10:13 + @test axiskeys(lstack(rboth), :a) == 'a':'c' + @test axiskeys(lstack(rboth), :b) == 10:13 nts = [(i=i, j="j", k=33) for i=1:3] - @test axiskeys(stack(nts), 1) == [:i, :j, :k] - @test axiskeys(stack(:z, nts...), 1) == [:i, :j, :k] - @test axiskeys(stack(n for n in nts), 1) == [:i, :j, :k] + @test axiskeys(lstack(nts), 1) == [:i, :j, :k] + @test axiskeys(lstack(:z, nts...), 1) == [:i, :j, :k] + @test axiskeys(lstack(n for n in nts), 1) == [:i, :j, :k] end @testset "dates" begin diff --git a/test/runtests.jl b/test/runtests.jl index 9cd94c5..c8bdc9b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,5 @@ using Test, AxisKeys, NamedDims -using Statistics, OffsetArrays, Tables, UniqueVectors, LazyStack +using Statistics, OffsetArrays, Tables, UniqueVectors using ChainRulesCore: ProjectTo, NoTangent using ChainRulesTestUtils: test_rrule using FiniteDifferences From 039c6776581bf4daacdea48227935921647790b3 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sat, 11 Feb 2023 19:28:21 -0500 Subject: [PATCH 32/37] v0.2.12 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index f00e2e5..1dcd845 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.11" +version = "0.2.12" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" From b98fcec8ff822c2ae3fb6de40a05c6db4aea0627 Mon Sep 17 00:00:00 2001 From: Oscar Smith Date: Thu, 27 Apr 2023 14:49:31 -0400 Subject: [PATCH 33/37] remove last `Vararg{<:...` --- src/statsbase.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/statsbase.jl b/src/statsbase.jl index d045e71..be2bac8 100644 --- a/src/statsbase.jl +++ b/src/statsbase.jl @@ -71,7 +71,7 @@ estimators = [ :AnalyticalNonlinearShrinkage, ] for estimator in estimators - @eval function Statistics.cov(ce::$estimator, A::KeyedMatrix, wv::Vararg{<:AbstractWeights}; dims=1, kwargs...) + @eval function Statistics.cov(ce::$estimator, A::KeyedMatrix, wv::Vararg{AbstractWeights}; dims=1, kwargs...) d = NamedDims.dim(A, dims) data = cov(ce, keyless_unname(A), wv...; dims=d, kwargs...) L1 = dimnames(A, 3 - d) From c335c9ac1df8bdbb05a74d9f22a5ca0612acd614 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Sat, 27 May 2023 03:53:03 +0200 Subject: [PATCH 34/37] Add support for StatsBase 0.34 --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 1dcd845..18c0d8b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.12" +version = "0.2.13" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" @@ -30,7 +30,7 @@ InvertedIndices = "1.0" LazyStack = "0.0.7, 0.0.8" NamedDims = "0.2.46, 0.3, 1" OffsetArrays = "0.10, 0.11, 1.0" -StatsBase = "0.32, 0.33" +StatsBase = "0.32, 0.33, 0.34" Tables = "0.2, 1" julia = "1.6" From a389a68abb225dd4b02b1b51c6550d08278f4c36 Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Sat, 2 Mar 2024 15:11:55 -0500 Subject: [PATCH 35/37] turn most deps into weakdeps (#151) * add 1.9 to CI * turn most deps into weakdeps * bump --- .github/workflows/ci.yml | 1 + Project.toml | 31 +++++++++++++++- src/fft.jl => ext/AbstractFFTsExt.jl | 8 +++- src/chainrules.jl => ext/ChainRulesCoreExt.jl | 5 +++ ext/CovarianceEstimationExt.jl | 33 +++++++++++++++++ ext/InvertedIndicesExt.jl | 10 +++++ src/stack.jl => ext/LazyStackExt.jl | 4 ++ ext/OffsetArraysExt.jl | 9 +++++ ext/StatisticsExt.jl | 37 +++++++++++++++++++ src/statsbase.jl => ext/StatsBaseExt.jl | 27 ++------------ src/AxisKeys.jl | 16 +++++--- src/functions.jl | 32 ---------------- src/selectors.jl | 7 ---- src/show.jl | 2 - src/wrap.jl | 4 +- 15 files changed, 148 insertions(+), 78 deletions(-) rename src/fft.jl => ext/AbstractFFTsExt.jl (96%) rename src/chainrules.jl => ext/ChainRulesCoreExt.jl (86%) create mode 100644 ext/CovarianceEstimationExt.jl create mode 100644 ext/InvertedIndicesExt.jl rename src/stack.jl => ext/LazyStackExt.jl (95%) create mode 100644 ext/OffsetArraysExt.jl create mode 100644 ext/StatisticsExt.jl rename src/statsbase.jl => ext/StatsBaseExt.jl (73%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 814d07a..e03383b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ jobs: matrix: version: - '1.6' + - '1.9' - '1' # Leave this line unchanged. '1' will automatically expand to the latest stable 1.x release of Julia. - 'nightly' os: diff --git a/Project.toml b/Project.toml index 18c0d8b..1b9cd00 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AxisKeys" uuid = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" license = "MIT" -version = "0.2.13" +version = "0.2.14" [deps] AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" @@ -17,6 +17,26 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +[weakdeps] +AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" +ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +CovarianceEstimation = "587fd27a-f159-11e8-2dae-1979310e6154" +InvertedIndices = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +LazyStack = "1fad7336-0346-5a1a-a56f-a06ba010965b" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" + +[extensions] +AbstractFFTsExt = "AbstractFFTs" +ChainRulesCoreExt = "ChainRulesCore" +CovarianceEstimationExt = "CovarianceEstimation" +InvertedIndicesExt = "InvertedIndices" +LazyStackExt = "LazyStack" +OffsetArraysExt = "OffsetArrays" +StatisticsExt = "Statistics" +StatsBaseExt = "StatsBase" + [compat] AbstractFFTs = "0.5, 1.0" BenchmarkTools = "0.5, 1.0" @@ -36,15 +56,22 @@ julia = "1.6" [extras] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a" +CovarianceEstimation = "587fd27a-f159-11e8-2dae-1979310e6154" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" +InvertedIndices = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +LazyStack = "1fad7336-0346-5a1a-a56f-a06ba010965b" NamedArrays = "86f7a689-2022-50b4-a561-43c23ac3c673" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" UniqueVectors = "2fbcfb34-fd0c-5fbb-b5d7-e826d8f5b0a9" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" [targets] -test = ["BenchmarkTools", "ChainRulesTestUtils", "DataFrames", "Dates", "FiniteDifferences", "FFTW", "NamedArrays", "Test", "UniqueVectors", "Unitful"] +test = ["BenchmarkTools", "CovarianceEstimation", "ChainRulesCore", "ChainRulesTestUtils", "DataFrames", "Dates", "FiniteDifferences", "FFTW", "InvertedIndices", "LazyStack", "NamedArrays", "OffsetArrays", "Test", "Statistics", "StatsBase", "UniqueVectors", "Unitful"] diff --git a/src/fft.jl b/ext/AbstractFFTsExt.jl similarity index 96% rename from src/fft.jl rename to ext/AbstractFFTsExt.jl index 1be5ce4..02f0dfe 100644 --- a/src/fft.jl +++ b/ext/AbstractFFTsExt.jl @@ -1,3 +1,7 @@ +module AbstractFFTsExt + +using AxisKeys: KeyedArray, NdaKa, axiskeys, keyless, NamedDims +using AbstractFFTs #= Simple support for FFTs using: @@ -7,8 +11,6 @@ Does not (yet) cover plan_fft & friends, because extracting the dimensions from those is tricky =# -using AbstractFFTs - for fun in [:fft, :ifft, :bfft, :rfft] @eval function AbstractFFTs.$fun(A::Union{KeyedArray,NdaKa}, dims = ntuple(+,ndims(A))) numerical_dims = NamedDims.dim(A, dims) @@ -80,3 +82,5 @@ function irfft_un_freq(x, len) s = inv(step(x) * len) range(zero(s), step = s, length = len) end + +end diff --git a/src/chainrules.jl b/ext/ChainRulesCoreExt.jl similarity index 86% rename from src/chainrules.jl rename to ext/ChainRulesCoreExt.jl index 8a2d872..68e7fe4 100644 --- a/src/chainrules.jl +++ b/ext/ChainRulesCoreExt.jl @@ -1,3 +1,6 @@ +module ChainRulesCoreExt + +using AxisKeys: KeyedArray, KaNda, NdaKa, keyless, keyless_unname, axiskeys, named_axiskeys, wrapdims using ChainRulesCore function ChainRulesCore.ProjectTo(x::Union{KaNda, NdaKa}) @@ -19,3 +22,5 @@ function ChainRulesCore.rrule(::typeof(keyless_unname), x) pb(y) = _KeyedArray_pullback(y, ProjectTo(x)) return keyless_unname(x), pb end + +end diff --git a/ext/CovarianceEstimationExt.jl b/ext/CovarianceEstimationExt.jl new file mode 100644 index 0000000..d00ec77 --- /dev/null +++ b/ext/CovarianceEstimationExt.jl @@ -0,0 +1,33 @@ +module CovarianceEstimationExt + +using AxisKeys: KeyedArray, KeyedMatrix, NamedDims, NamedDimsArray, axiskeys, dimnames, keyless_unname, hasnames +using CovarianceEstimation +using CovarianceEstimation: AbstractWeights +using CovarianceEstimation.Statistics + +# Since we get ambiguity errors with specific implementations we need to wrap each supported method +# A better approach might be to add `NamedDims` support to CovarianceEstimators.jl in the future. + +estimators = [ + :SimpleCovariance, + :LinearShrinkage, + :DiagonalUnitVariance, + :DiagonalCommonVariance, + :DiagonalUnequalVariance, + :CommonCovariance, + :PerfectPositiveCorrelation, + :ConstantCorrelation, + :AnalyticalNonlinearShrinkage, +] +for estimator in estimators + @eval function Statistics.cov(ce::$estimator, A::KeyedMatrix, wv::Vararg{AbstractWeights}; dims=1, kwargs...) + d = NamedDims.dim(A, dims) + data = cov(ce, keyless_unname(A), wv...; dims=d, kwargs...) + L1 = dimnames(A, 3 - d) + data2 = hasnames(A) ? NamedDimsArray(data, (L1, L1)) : data + K1 = axiskeys(A, 3 - d) + KeyedArray(data2, (copy(K1), copy(K1))) + end +end + +end diff --git a/ext/InvertedIndicesExt.jl b/ext/InvertedIndicesExt.jl new file mode 100644 index 0000000..9d30e2d --- /dev/null +++ b/ext/InvertedIndicesExt.jl @@ -0,0 +1,10 @@ +module InvertedIndicesExt + +using AxisKeys +using InvertedIndices + +# needs only Base.to_indices in struct.jl to work, +# plus this to work when used in round brackets: +AxisKeys.findindex(not::InvertedIndex, r::AbstractVector) = Base.unalias(r, not) + +end diff --git a/src/stack.jl b/ext/LazyStackExt.jl similarity index 95% rename from src/stack.jl rename to ext/LazyStackExt.jl index 1849591..4c202dc 100644 --- a/src/stack.jl +++ b/ext/LazyStackExt.jl @@ -1,4 +1,6 @@ +module LazyStackExt +using AxisKeys: KeyedArray, NamedDims, NamedDimsArray, axiskeys, hasnames, dimnames, keys_or_axes import LazyStack # for stack_iter @@ -57,3 +59,5 @@ function LazyStack.getnames(xs::AbstractArray{<:KeyedArray{T,N,IT}}) where {T,N, out_names = hasnames(xs) ? dimnames(xs) : NamedDims.dimnames(xs) (NamedDims.dimnames(IT)..., out_names...) end + +end diff --git a/ext/OffsetArraysExt.jl b/ext/OffsetArraysExt.jl new file mode 100644 index 0000000..56eec09 --- /dev/null +++ b/ext/OffsetArraysExt.jl @@ -0,0 +1,9 @@ +module OffsetArraysExt + +using AxisKeys +using OffsetArrays + +AxisKeys.no_offset(x::OffsetArray) = parent(x) +AxisKeys.shorttype(r::OffsetArray) = "OffsetArray(::" * shorttype(parent(r)) * ",...)" + +end diff --git a/ext/StatisticsExt.jl b/ext/StatisticsExt.jl new file mode 100644 index 0000000..c85e15f --- /dev/null +++ b/ext/StatisticsExt.jl @@ -0,0 +1,37 @@ +module StatisticsExt + +using AxisKeys: KeyedArray, KeyedMatrix, NamedDims, axiskeys +using Statistics + +for fun in [:mean, :std, :var] # These don't use mapreduce, but could perhaps be handled better? + @eval function Statistics.$fun(A::KeyedArray; dims=:, kwargs...) + dims === Colon() && return $fun(parent(A); kwargs...) + numerical_dims = NamedDims.dim(A, dims) + data = $fun(parent(A); dims=numerical_dims, kwargs...) + new_keys = ntuple(d -> d in numerical_dims ? Base.OneTo(1) : axiskeys(A,d), ndims(A)) + return KeyedArray(data, map(copy, new_keys))#, copy(A.meta)) + end +end + +# Handle function interface for `mean` only +if VERSION >= v"1.3" + @eval function Statistics.mean(f, A::KeyedArray; dims=:, kwargs...) + dims === Colon() && return mean(f, parent(A); kwargs...) + numerical_dims = NamedDims.dim(A, dims) + data = mean(f, parent(A); dims=numerical_dims, kwargs...) + new_keys = ntuple(d -> d in numerical_dims ? Base.OneTo(1) : axiskeys(A,d), ndims(A)) + return KeyedArray(data, map(copy, new_keys))#, copy(A.meta)) + end +end + +for fun in [:cov, :cor] # Returned the axes work are different for cov and cor + @eval function Statistics.$fun(A::KeyedMatrix; dims=1, kwargs...) + numerical_dim = NamedDims.dim(A, dims) + data = $fun(parent(A); dims=numerical_dim, kwargs...) + # Use same remaining axis for both dimensions of data + rem_key = axiskeys(A, 3-numerical_dim) + KeyedArray(data, (copy(rem_key), copy(rem_key))) + end +end + +end diff --git a/src/statsbase.jl b/ext/StatsBaseExt.jl similarity index 73% rename from src/statsbase.jl rename to ext/StatsBaseExt.jl index be2bac8..25a6d9e 100644 --- a/src/statsbase.jl +++ b/ext/StatsBaseExt.jl @@ -1,5 +1,8 @@ +module StatsBaseExt +using AxisKeys: KeyedArray, KeyedMatrix, NamedDims, NamedDimsArray, axiskeys, dimnames, keyless_unname, hasnames using StatsBase +using StatsBase.Statistics # Support some of the weighted statistics function in StatsBase # NOTES: @@ -55,28 +58,4 @@ for fun in (:std, :var, :cov) ) end -# Since we get ambiguity errors with specific implementations we need to wrap each supported method -# A better approach might be to add `NamedDims` support to CovarianceEstimators.jl in the future. -using CovarianceEstimation - -estimators = [ - :SimpleCovariance, - :LinearShrinkage, - :DiagonalUnitVariance, - :DiagonalCommonVariance, - :DiagonalUnequalVariance, - :CommonCovariance, - :PerfectPositiveCorrelation, - :ConstantCorrelation, - :AnalyticalNonlinearShrinkage, -] -for estimator in estimators - @eval function Statistics.cov(ce::$estimator, A::KeyedMatrix, wv::Vararg{AbstractWeights}; dims=1, kwargs...) - d = NamedDims.dim(A, dims) - data = cov(ce, keyless_unname(A), wv...; dims=d, kwargs...) - L1 = dimnames(A, 3 - d) - data2 = hasnames(A) ? NamedDimsArray(data, (L1, L1)) : data - K1 = axiskeys(A, 3 - d) - KeyedArray(data2, (copy(K1), copy(K1))) - end end diff --git a/src/AxisKeys.jl b/src/AxisKeys.jl index adeabcb..1914775 100644 --- a/src/AxisKeys.jl +++ b/src/AxisKeys.jl @@ -26,11 +26,15 @@ include("show.jl") include("tables.jl") # Tables.jl -include("stack.jl") # LazyStack.jl - -include("fft.jl") # AbstractFFTs.jl - -include("statsbase.jl") # StatsBase.jl +if !isdefined(Base, :get_extension) + include("../ext/AbstractFFTsExt.jl") + include("../ext/ChainRulesCoreExt.jl") + include("../ext/CovarianceEstimationExt.jl") + include("../ext/InvertedIndicesExt.jl") + include("../ext/LazyStackExt.jl") + include("../ext/OffsetArraysExt.jl") + include("../ext/StatisticsExt.jl") + include("../ext/StatsBaseExt.jl") +end -include("chainrules.jl") end diff --git a/src/functions.jl b/src/functions.jl index f1758f8..a4b10f5 100644 --- a/src/functions.jl +++ b/src/functions.jl @@ -43,38 +43,6 @@ function Base.mapreduce(f, op, A::KeyedArray; dims=:, kwargs...) # sum, prod, et return KeyedArray(data, map(copy, new_keys))#, copy(A.meta)) end -using Statistics -for fun in [:mean, :std, :var] # These don't use mapreduce, but could perhaps be handled better? - @eval function Statistics.$fun(A::KeyedArray; dims=:, kwargs...) - dims === Colon() && return $fun(parent(A); kwargs...) - numerical_dims = NamedDims.dim(A, dims) - data = $fun(parent(A); dims=numerical_dims, kwargs...) - new_keys = ntuple(d -> d in numerical_dims ? Base.OneTo(1) : axiskeys(A,d), ndims(A)) - return KeyedArray(data, map(copy, new_keys))#, copy(A.meta)) - end -end - -# Handle function interface for `mean` only -if VERSION >= v"1.3" - @eval function Statistics.mean(f, A::KeyedArray; dims=:, kwargs...) - dims === Colon() && return mean(f, parent(A); kwargs...) - numerical_dims = NamedDims.dim(A, dims) - data = mean(f, parent(A); dims=numerical_dims, kwargs...) - new_keys = ntuple(d -> d in numerical_dims ? Base.OneTo(1) : axiskeys(A,d), ndims(A)) - return KeyedArray(data, map(copy, new_keys))#, copy(A.meta)) - end -end - -for fun in [:cov, :cor] # Returned the axes work are different for cov and cor - @eval function Statistics.$fun(A::KeyedMatrix; dims=1, kwargs...) - numerical_dim = NamedDims.dim(A, dims) - data = $fun(parent(A); dims=numerical_dim, kwargs...) - # Use same remaining axis for both dimensions of data - rem_key = axiskeys(A, 3-numerical_dim) - KeyedArray(data, (copy(rem_key), copy(rem_key))) - end -end - function Base.dropdims(A::KeyedArray; dims) numerical_dims = NamedDims.dim(A, dims) data = dropdims(parent(A); dims=dims) diff --git a/src/selectors.jl b/src/selectors.jl index 62f5917..bd51f12 100644 --- a/src/selectors.jl +++ b/src/selectors.jl @@ -1,10 +1,3 @@ - -using InvertedIndices -# needs only Base.to_indices in struct.jl to work, -# plus this to work when used in round brackets: - -findindex(not::InvertedIndex, r::AbstractVector) = Base.unalias(r, not) - using IntervalSets findindex(int::Interval, r::AbstractVector) = diff --git a/src/show.jl b/src/show.jl index 1821039..6aca95a 100644 --- a/src/show.jl +++ b/src/show.jl @@ -26,7 +26,6 @@ end shorttype(r::Vector{T}) where {T} = "Vector{$T}" shorttype(r::OneTo) = "OneTo{Int}" shorttype(r::SubArray) = "view(::" * shorttype(parent(r)) * ",...)" -shorttype(r::OffsetArray) = "OffsetArray(::" * shorttype(parent(r)) * ",...)" function shorttype(r) bits = split(string(typeof(r)),',') length(bits) == 1 && return bits[1] @@ -122,7 +121,6 @@ function keyed_print_matrix(io::IO, A, reduce_size::Bool=false) end no_offset(x) = x -no_offset(x::OffsetArray) = parent(x) full(x::DenseArray) = x full(x::AbstractArray) = collect(x) # deal with sparse diff --git a/src/wrap.jl b/src/wrap.jl index 58fe258..6a2d4ea 100644 --- a/src/wrap.jl +++ b/src/wrap.jl @@ -53,8 +53,6 @@ for fast lookup. wrapdims(A::AbstractArray, T::Type, r::Union{AbstractVector,Nothing}, keys::Union{AbstractVector,Nothing}...) = KeyedArray(A, map(T, check_keys(A, (r, keys...)))) -using OffsetArrays - function check_keys(A, keys) ndims(A) == length(keys) || throw(ArgumentError( "wrong number of key vectors, got $(length(keys)) with ndims(A) == $(ndims(A))")) @@ -65,7 +63,7 @@ function check_keys(A, keys) elseif axes(r,1) == axes(A,d) r elseif length(r) == size(A,d) - OffsetArray(r, axes(A,d)) + reshape(r, Base.IdentityUnitRange(axes(A,d))) elseif r isa AbstractRange l = size(A,d) r′ = extend_range(r, l) From c924f96e44431db442202c60fa9a4bf2f87462cb Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Sat, 2 Mar 2024 14:12:41 -0600 Subject: [PATCH 36/37] add tests for sorting with non int eltypes (#149) --- test/_functions.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/_functions.jl b/test/_functions.jl index 189bf1e..d4ad2ee 100644 --- a/test/_functions.jl +++ b/test/_functions.jl @@ -4,6 +4,7 @@ using NamedDims: unname M = wrapdims(rand(Int8, 3,4), r='a':'c', c=2:5) MN = NamedDimsArray(M.data.data, r='a':'c', c=2:5) V = wrapdims(rand(1:99, 10), v=10:10:100) + VN = NamedDimsArray(V.data.data, v=10:10:100) A3 = wrapdims(rand(Int8, 3,4,2), r='a':'c', c=2:5, p=[10.0, 20.0]) @@ -87,6 +88,9 @@ end @testset "sort & reverse" begin @test sort(V)(20) == V(20) + # need to test with non integer eltypes: + @test sort!(float.(V))(20) == V(20) + @test first(sort!(float.(A3); dims=1)) == first(sort(A3; dims=1)) @test axiskeys(sort(M, dims=:c), :c) isa Base.OneTo @test axiskeys(sort(M, dims=:c), :r) == 'a':'c' From 3eafa4df2755d5d89afec0426452b45aa508c102 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sat, 2 Mar 2024 17:28:30 -0500 Subject: [PATCH 37/37] Revert "add tests for sorting with non int eltypes (#149)" This reverts commit c924f96e44431db442202c60fa9a4bf2f87462cb. --- test/_functions.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/_functions.jl b/test/_functions.jl index d4ad2ee..189bf1e 100644 --- a/test/_functions.jl +++ b/test/_functions.jl @@ -4,7 +4,6 @@ using NamedDims: unname M = wrapdims(rand(Int8, 3,4), r='a':'c', c=2:5) MN = NamedDimsArray(M.data.data, r='a':'c', c=2:5) V = wrapdims(rand(1:99, 10), v=10:10:100) - VN = NamedDimsArray(V.data.data, v=10:10:100) A3 = wrapdims(rand(Int8, 3,4,2), r='a':'c', c=2:5, p=[10.0, 20.0]) @@ -88,9 +87,6 @@ end @testset "sort & reverse" begin @test sort(V)(20) == V(20) - # need to test with non integer eltypes: - @test sort!(float.(V))(20) == V(20) - @test first(sort!(float.(A3); dims=1)) == first(sort(A3; dims=1)) @test axiskeys(sort(M, dims=:c), :c) isa Base.OneTo @test axiskeys(sort(M, dims=:c), :r) == 'a':'c'