From 7980d4b53b7508318373c61911b4f32dc17be160 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Wed, 8 Jan 2025 09:43:51 +0100 Subject: [PATCH] adding examples in the initializer docs --- Project.toml | 2 +- src/esn/esn_input_layers.jl | 234 +++++++++++++++----------- src/esn/esn_reservoirs.jl | 316 ++++++++++++++++++++++-------------- 3 files changed, 328 insertions(+), 224 deletions(-) diff --git a/Project.toml b/Project.toml index f939bba1..3953f3b8 100644 --- a/Project.toml +++ b/Project.toml @@ -41,7 +41,7 @@ SafeTestsets = "0.1" Statistics = "1.10" StatsBase = "0.34.4" Test = "1" -WeightInitializers = "1.0.4" +WeightInitializers = "1.0.5" julia = "1.10" [extras] diff --git a/src/esn/esn_input_layers.jl b/src/esn/esn_input_layers.jl index 04487f96..66224bab 100644 --- a/src/esn/esn_input_layers.jl +++ b/src/esn/esn_input_layers.jl @@ -1,29 +1,37 @@ """ - scaled_rand(rng::AbstractRNG, ::Type{T}, dims::Integer...; scaling=T(0.1)) where {T <: Number} + scaled_rand([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + scaling=0.1) -Create and return a matrix with random values, uniformly distributed within a range defined by `scaling`. This function is useful for initializing matrices, such as the layers of a neural network, with scaled random values. +Create and return a matrix with random values, uniformly distributed within +a range defined by `scaling`. # Arguments - - `rng`: An instance of `AbstractRNG` for random number generation. - - `T`: The data type for the elements of the matrix. - - `dims`: Dimensions of the matrix. It must be a 2-element tuple specifying the number of rows and columns (e.g., `(res_size, in_size)`). - - `scaling`: A scaling factor to define the range of the uniform distribution. The matrix elements will be randomly chosen from the range `[-scaling, scaling]`. Defaults to `T(0.1)`. - -# Returns - -A matrix of type with dimensions specified by `dims`. Each element of the matrix is a random number uniformly distributed between `-scaling` and `scaling`. - -# Example - -```julia -rng = Random.default_rng() -matrix = scaled_rand(rng, Float64, (100, 50); scaling=0.2) + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. + Default is `Float32`. + - `dims`: Dimensions of the matrix. Should follow `res_size x in_size`. + - `scaling`: A scaling factor to define the range of the uniform distribution. + The matrix elements will be randomly chosen from the + range `[-scaling, scaling]`. Defaults to `0.1`. + +# Examples + +```jldoctest +julia> res_input = scaled_rand(8, 3) +8×3 Matrix{Float32}: + -0.0669356 -0.0292692 -0.0188943 + 0.0159724 0.004071 -0.0737949 + 0.026355 -0.0191563 0.0714962 + -0.0177412 0.0279123 0.0892906 + -0.0184405 0.0567368 0.0190222 + 0.0944272 0.0679244 0.0148647 + -0.0799005 -0.0891089 -0.0444782 + -0.0970182 0.0934286 0.03553 ``` """ -function scaled_rand(rng::AbstractRNG, - ::Type{T}, - dims::Integer...; +function scaled_rand(rng::AbstractRNG, ::Type{T}, dims::Integer...; scaling=T(0.1)) where {T <: Number} res_size, in_size = dims layer_matrix = (DeviceAgnostic.rand(rng, T, res_size, in_size) .- T(0.5)) .* @@ -32,37 +40,42 @@ function scaled_rand(rng::AbstractRNG, end """ - weighted_init(rng::AbstractRNG, ::Type{T}, dims::Integer...; scaling=T(0.1)) where {T <: Number} + weighted_init([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + scaling=0.1) -Create and return a matrix representing a weighted input layer for Echo State Networks (ESNs). This initializer generates a weighted input matrix with random non-zero elements distributed uniformly within the range [-`scaling`, `scaling`], inspired by the approach in [^Lu]. +Create and return a matrix representing a weighted input layer. +This initializer generates a weighted input matrix with random non-zero +elements distributed uniformly within the range [-`scaling`, `scaling`] [^Lu2017]. # Arguments - - `rng`: An instance of `AbstractRNG` for random number generation. - - `T`: The data type for the elements of the matrix. - - `dims`: A 2-element tuple specifying the approximate reservoir size and input size (e.g., `(approx_res_size, in_size)`). - - `scaling`: The scaling factor for the weight distribution. Defaults to `T(0.1)`. - -# Returns - -A matrix representing the weighted input layer as defined in [^Lu2017]. The matrix dimensions will be adjusted to ensure each input unit connects to an equal number of reservoir units. - -# Example - -```julia -rng = Random.default_rng() -input_layer = weighted_init(rng, Float64, (3, 300); scaling=0.2) + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. + Default is `Float32`. + - `dims`: Dimensions of the matrix. Should follow `res_size x in_size`. + - `scaling`: The scaling factor for the weight distribution. + Defaults to `0.1`. + +# Examples + +```jldoctest +julia> res_input = weighted_init(8, 3) +6×3 Matrix{Float32}: + 0.0452399 0.0 0.0 + -0.0348047 0.0 0.0 + 0.0 -0.0386004 0.0 + 0.0 0.00981022 0.0 + 0.0 0.0 0.0577838 + 0.0 0.0 -0.0562827 ``` -# References - [^Lu2017]: Lu, Zhixin, et al. - "Reservoir observers: Model-free inference of unmeasured variables in chaotic systems." + "Reservoir observers: Model-free inference of unmeasured variables in + chaotic systems." Chaos: An Interdisciplinary Journal of Nonlinear Science 27.4 (2017): 041102. """ -function weighted_init(rng::AbstractRNG, - ::Type{T}, - dims::Integer...; +function weighted_init(rng::AbstractRNG, ::Type{T}, dims::Integer...; scaling=T(0.1)) where {T <: Number} approx_res_size, in_size = dims res_size = Int(floor(approx_res_size / in_size) * in_size) @@ -78,31 +91,24 @@ function weighted_init(rng::AbstractRNG, end """ - informed_init(rng::AbstractRNG, ::Type{T}, dims::Integer...; scaling=T(0.1), model_in_size, gamma=T(0.5)) where {T <: Number} + informed_init([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + scaling=0.1, model_in_size, gamma=0.5) -Create a layer of a neural network. +Create an input layer for informed echo state networks. # Arguments - - `rng::AbstractRNG`: The random number generator. - - `T::Type`: The data type. - - `dims::Integer...`: The dimensions of the layer. - - `scaling::T = T(0.1)`: The scaling factor for the input matrix. + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. + Default is `Float32`. + - `dims`: Dimensions of the matrix. Should follow `res_size x in_size`. + - `scaling`: The scaling factor for the input matrix. + Default is 0.1. - `model_in_size`: The size of the input model. - - `gamma::T = T(0.5)`: The gamma value. + - `gamma`: The gamma value. Default is 0.5. -# Returns - - - `input_matrix`: The created input matrix for the layer. - -# Example - -```julia -rng = Random.default_rng() -dims = (100, 200) -model_in_size = 50 -input_matrix = informed_init(rng, Float64, dims; model_in_size=model_in_size) -``` +# Examples """ function informed_init(rng::AbstractRNG, ::Type{T}, dims::Integer...; scaling=T(0.1), model_in_size, gamma=T(0.5)) where {T <: Number} @@ -141,41 +147,79 @@ function informed_init(rng::AbstractRNG, ::Type{T}, dims::Integer...; end """ - irrational_sample_init(rng::AbstractRNG, ::Type{T}, dims::Integer...; - weight = 0.1, - sampling = IrrationalSample(; irrational = pi, start = 1) - ) where {T <: Number} + minimal_init([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + sampling_type=:bernoulli, weight=0.1, irrational=pi, start=1, p=0.5) -Create a layer matrix using the provided random number generator and sampling parameters. +Create a layer matrix with uniform weights determined by `weight`. The sign difference +is randomly determined by the `sampling` chosen. # Arguments - - `rng::AbstractRNG`: The random number generator used to generate random numbers. - - `dims::Integer...`: The dimensions of the layer matrix. + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. + Default is `Float32`. + - `dims`: Dimensions of the matrix. Should follow `res_size x in_size`. - `weight`: The weight used to fill the layer matrix. Default is 0.1. - - `sampling`: The sampling parameters used to generate the input matrix. Default is IrrationalSample(irrational = pi, start = 1). - -# Returns - -The layer matrix generated using the provided random number generator and sampling parameters. - -# Example - -```julia -using Random -rng = Random.default_rng() -dims = (3, 2) -weight = 0.5 -layer_matrix = irrational_sample_init(rng, Float64, dims; weight=weight, - sampling=IrrationalSample(; irrational=sqrt(2), start=1)) + - `sampling_type`: The sampling parameters used to generate the input matrix. + Default is `:bernoulli`. + - `irrational`: Irrational number chosen for sampling if `sampling_type=:irrational`. + Default is `pi`. + - `start`: Starting value for the irrational sample. Default is 1 + - `p`: Probability for the Bernoulli sampling. Lower probability increases negative + value. Higher probability increases positive values. Default is 0.5 + +# Examples + +```jldoctest +julia> res_input = minimal_init(8, 3) +8×3 Matrix{Float32}: + 0.1 -0.1 0.1 + -0.1 0.1 0.1 + -0.1 -0.1 0.1 + -0.1 -0.1 -0.1 + 0.1 0.1 0.1 + -0.1 -0.1 -0.1 + -0.1 -0.1 0.1 + 0.1 -0.1 0.1 + +julia> res_input = minimal_init(8, 3; sampling_type=:irrational) +8×3 Matrix{Float32}: + -0.1 0.1 -0.1 + 0.1 -0.1 -0.1 + 0.1 0.1 -0.1 + 0.1 0.1 0.1 + -0.1 -0.1 -0.1 + 0.1 0.1 0.1 + 0.1 0.1 -0.1 + -0.1 0.1 -0.1 + +julia> res_input = minimal_init(8, 3; p=0.1) # lower p -> more negative signs +8×3 Matrix{Float32}: + -0.1 -0.1 -0.1 + -0.1 -0.1 -0.1 + -0.1 -0.1 -0.1 + -0.1 -0.1 -0.1 + 0.1 -0.1 -0.1 + -0.1 -0.1 -0.1 + -0.1 -0.1 -0.1 + -0.1 -0.1 -0.1 + +julia> res_input = minimal_init(8, 3; p=0.8)# higher p -> more positive signs +8×3 Matrix{Float32}: + 0.1 0.1 0.1 + -0.1 0.1 0.1 + -0.1 0.1 0.1 + 0.1 0.1 0.1 + 0.1 0.1 0.1 + 0.1 -0.1 0.1 + -0.1 0.1 0.1 + 0.1 0.1 0.1 ``` """ function minimal_init(rng::AbstractRNG, ::Type{T}, dims::Integer...; - sampling_type::Symbol=:bernoulli, - weight::Number=T(0.1), - irrational::Real=pi, - start::Int=1, - p::Number=T(0.5)) where {T <: Number} + sampling_type::Symbol=:bernoulli, weight::Number=T(0.1), irrational::Real=pi, + start::Int=1, p::Number=T(0.5)) where {T <: Number} res_size, in_size = dims if sampling_type == :bernoulli layer_matrix = _create_bernoulli(p, res_size, in_size, weight, rng, T) @@ -193,12 +237,8 @@ function minimal_init(rng::AbstractRNG, ::Type{T}, dims::Integer...; return layer_matrix end -function _create_bernoulli(p::Number, - res_size::Int, - in_size::Int, - weight::Number, - rng::AbstractRNG, - ::Type{T}) where {T <: Number} +function _create_bernoulli(p::Number, res_size::Int, in_size::Int, weight::Number, + rng::AbstractRNG, ::Type{T}) where {T <: Number} input_matrix = DeviceAgnostic.zeros(rng, T, res_size, in_size) for i in 1:res_size for j in 1:in_size @@ -212,12 +252,8 @@ function _create_bernoulli(p::Number, return input_matrix end -function _create_irrational(irrational::Irrational, - start::Int, - res_size::Int, - in_size::Int, - weight::Number, - rng::AbstractRNG, +function _create_irrational(irrational::Irrational, start::Int, res_size::Int, + in_size::Int, weight::Number, rng::AbstractRNG, ::Type{T}) where {T <: Number} setprecision(BigFloat, Int(ceil(log2(10) * (res_size * in_size + start + 1)))) ir_string = string(BigFloat(irrational)) |> collect diff --git a/src/esn/esn_reservoirs.jl b/src/esn/esn_reservoirs.jl index 36276099..cf9efd06 100644 --- a/src/esn/esn_reservoirs.jl +++ b/src/esn/esn_reservoirs.jl @@ -1,30 +1,37 @@ """ - rand_sparse(rng::AbstractRNG, ::Type{T}, dims::Integer...; radius=1.0, sparsity=0.1) + rand_sparse([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + radius=1.0, sparsity=0.1, std=1.0) -Create and return a random sparse reservoir matrix for use in Echo State Networks (ESNs). The matrix will be of size specified by `dims`, with specified `sparsity` and scaled spectral radius according to `radius`. +Create and return a random sparse reservoir matrix. +The matrix will be of size specified by `dims`, with specified `sparsity` +and scaled spectral radius according to `radius`. # Arguments - - `rng`: An instance of `AbstractRNG` for random number generation. - - `T`: The data type for the elements of the matrix. + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. + Default is `Float32`. - `dims`: Dimensions of the reservoir matrix. - - `radius`: The desired spectral radius of the reservoir. Defaults to 1.0. - - `sparsity`: The sparsity level of the reservoir matrix, controlling the fraction of zero elements. Defaults to 0.1. - -# Returns - -A matrix representing the random sparse reservoir. - -# References - -This type of reservoir initialization is commonly used in ESNs for capturing temporal dependencies in data. + - `radius`: The desired spectral radius of the reservoir. + Defaults to 1.0. + - `sparsity`: The sparsity level of the reservoir matrix, + controlling the fraction of zero elements. Defaults to 0.1. + +# Examples + +```jldoctest +julia> res_matrix = rand_sparse(5, 5; sparsity=0.5) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 0.0 + 0.0 0.794565 0.0 0.26164 0.0 + 0.0 0.0 -0.931294 0.0 0.553706 + 0.723235 -0.524727 0.0 0.0 0.0 + 1.23723 0.0 0.181824 -1.5478 0.465328 +``` """ -function rand_sparse(rng::AbstractRNG, - ::Type{T}, - dims::Integer...; - radius=T(1.0), - sparsity=T(0.1), - std=T(1.0)) where {T <: Number} +function rand_sparse(rng::AbstractRNG, ::Type{T}, dims::Integer...; + radius=T(1.0), sparsity=T(0.1), std=T(1.0)) where {T <: Number} lcl_sparsity = T(1) - sparsity #consistency with current implementations reservoir_matrix = sparse_init(rng, T, dims...; sparsity=lcl_sparsity, std=std) rho_w = maximum(abs.(eigvals(reservoir_matrix))) @@ -36,38 +43,49 @@ function rand_sparse(rng::AbstractRNG, end """ - delay_line(rng::AbstractRNG, ::Type{T}, dims::Integer...; weight=0.1) where {T <: Number} + delay_line([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + weight=0.1) -Create and return a delay line reservoir matrix for use in Echo State Networks (ESNs). A delay line reservoir is a deterministic structure where each unit is connected only to its immediate predecessor with a specified weight. This method is particularly useful for tasks that require specific temporal processing. +Create and return a delay line reservoir matrix [^Rodan2010]. # Arguments - - `rng`: An instance of `AbstractRNG` for random number generation. This argument is not used in the current implementation but is included for consistency with other initialization functions. - - `T`: The data type for the elements of the matrix. - - `dims`: Dimensions of the reservoir matrix. Typically, this should be a tuple of two equal integers representing a square matrix. - - `weight`: The weight determines the absolute value of all connections in the reservoir. Defaults to 0.1. - -# Returns - -A delay line reservoir matrix with dimensions specified by `dims`. The matrix is initialized such that each element in the `i+1`th row and `i`th column is set to `weight`, and all other elements are zeros. - -# Example - -```julia -reservoir = delay_line(Float64, 100, 100; weight=0.2) + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. + Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + - `weight`: Determines the value of all connections in the reservoir. + Default is 0.1. + +# Examples + +```jldoctest +julia> res_matrix = delay_line(5, 5) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 0.0 + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.0 0.0 0.0 + 0.0 0.0 0.1 0.0 0.0 + 0.0 0.0 0.0 0.1 0.0 + +julia> res_matrix = delay_line(5, 5; weight=1) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 0.0 + 1.0 0.0 0.0 0.0 0.0 + 0.0 1.0 0.0 0.0 0.0 + 0.0 0.0 1.0 0.0 0.0 + 0.0 0.0 0.0 1.0 0.0 ``` -# References - -This type of reservoir initialization is described in: -Rodan, Ali, and Peter Tino. "Minimum complexity echo state network." IEEE Transactions on Neural Networks 22.1 (2010): 131-144. +[^Rodan2010]: Rodan, Ali, and Peter Tino. "Minimum complexity echo state network." + IEEE transactions on neural networks 22.1 (2010): 131-144. """ -function delay_line(rng::AbstractRNG, - ::Type{T}, - dims::Integer...; +function delay_line(rng::AbstractRNG, ::Type{T}, dims::Integer...; weight=T(0.1)) where {T <: Number} reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) - @assert length(dims) == 2&&dims[1] == dims[2] "The dimensions must define a square matrix (e.g., (100, 100))" + @assert length(dims) == 2&&dims[1] == dims[2] "The dimensions + must define a square matrix (e.g., (100, 100))" for i in 1:(dims[1] - 1) reservoir_matrix[i + 1, i] = weight @@ -77,35 +95,49 @@ function delay_line(rng::AbstractRNG, end """ - delay_line_backward(rng::AbstractRNG, ::Type{T}, dims::Integer...; - weight = T(0.1), fb_weight = T(0.2)) where {T <: Number} + delay_line_backward([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + weight = 0.1, fb_weight = 0.2) -Create a delay line backward reservoir with the specified by `dims` and weights. Creates a matrix with backward connections -as described in [^Rodan2010]. The `weight` and `fb_weight` can be passed as either arguments or -keyword arguments, and they determine the absolute values of the connections in the reservoir. +Create a delay line backward reservoir with the specified by `dims` and weights. +Creates a matrix with backward connections as described in [^Rodan2010]. # Arguments - - `rng::AbstractRNG`: Random number generator. - - `T::Type`: Type of the elements in the reservoir matrix. - - `dims::Integer...`: Dimensions of the reservoir matrix. - - `weight::T`: The weight determines the absolute value of forward connections in the reservoir, and is set to 0.1 by default. - - `fb_weight::T`: The `fb_weight` determines the absolute value of backward connections in the reservoir, and is set to 0.2 by default. - -# Returns - -Reservoir matrix with the dimensions specified by `dims` and weights. - -# References + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. + Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + - `weight`: The weight determines the absolute value of + forward connections in the reservoir. Default is 0.1 + - `fb_weight`: Determines the absolute value of backward connections + in the reservoir. Default is 0.2 + +# Examples + +```jldoctest +julia> res_matrix = delay_line_backward(5, 5) +5×5 Matrix{Float32}: + 0.0 0.2 0.0 0.0 0.0 + 0.1 0.0 0.2 0.0 0.0 + 0.0 0.1 0.0 0.2 0.0 + 0.0 0.0 0.1 0.0 0.2 + 0.0 0.0 0.0 0.1 0.0 + +julia> res_matrix = delay_line_backward(Float16, 5, 5) +5×5 Matrix{Float16}: + 0.0 0.2 0.0 0.0 0.0 + 0.1 0.0 0.2 0.0 0.0 + 0.0 0.1 0.0 0.2 0.0 + 0.0 0.0 0.1 0.0 0.2 + 0.0 0.0 0.0 0.1 0.0 +``` [^Rodan2010]: Rodan, Ali, and Peter Tino. "Minimum complexity echo state network." IEEE transactions on neural networks 22.1 (2010): 131-144. """ -function delay_line_backward(rng::AbstractRNG, - ::Type{T}, - dims::Integer...; - weight=T(0.1), - fb_weight=T(0.2)) where {T <: Number} +function delay_line_backward(rng::AbstractRNG, ::Type{T}, dims::Integer...; + weight=T(0.1), fb_weight=T(0.2)) where {T <: Number} res_size = first(dims) reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) @@ -118,34 +150,51 @@ function delay_line_backward(rng::AbstractRNG, end """ - cycle_jumps(rng::AbstractRNG, ::Type{T}, dims::Integer...; - cycle_weight = T(0.1), jump_weight = T(0.1), jump_size = 3) where {T <: Number} + cycle_jumps([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + cycle_weight = 0.1, jump_weight = 0.1, jump_size = 3) -Create a cycle jumps reservoir with the specified dimensions, cycle weight, jump weight, and jump size. +Create a cycle jumps reservoir with the specified dimensions, +cycle weight, jump weight, and jump size. # Arguments - - `rng::AbstractRNG`: Random number generator. - - `T::Type`: Type of the elements in the reservoir matrix. - - `dims::Integer...`: Dimensions of the reservoir matrix. - - `cycle_weight::T = T(0.1)`: The weight of cycle connections. - - `jump_weight::T = T(0.1)`: The weight of jump connections. - - `jump_size::Int = 3`: The number of steps between jump connections. - -# Returns - -Reservoir matrix with the specified dimensions, cycle weight, jump weight, and jump size. - -# References + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. + Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + - `cycle_weight`: The weight of cycle connections. + Default is 0.1. + - `jump_weight`: The weight of jump connections. + Default is 0.1. + - `jump_size`: The number of steps between jump connections. + Default is 3. + +# Examples + +```jldoctest +julia> res_matrix = cycle_jumps(5, 5) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.1 0.1 + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.0 0.0 0.0 + 0.1 0.0 0.1 0.0 0.0 + 0.0 0.0 0.0 0.1 0.0 + +julia> res_matrix = cycle_jumps(5, 5; jump_size=2) +5×5 Matrix{Float32}: + 0.0 0.0 0.1 0.0 0.1 + 0.1 0.0 0.0 0.0 0.0 + 0.1 0.1 0.0 0.0 0.1 + 0.0 0.0 0.1 0.0 0.0 + 0.0 0.0 0.1 0.1 0.0 +``` [^Rodan2012]: Rodan, Ali, and Peter Tiňo. "Simple deterministically constructed cycle reservoirs with regular jumps." Neural computation 24.7 (2012): 1822-1852. """ -function cycle_jumps(rng::AbstractRNG, - ::Type{T}, - dims::Integer...; - cycle_weight::Number=T(0.1), - jump_weight::Number=T(0.1), +function cycle_jumps(rng::AbstractRNG, ::Type{T}, dims::Integer...; + cycle_weight::Number=T(0.1), jump_weight::Number=T(0.1), jump_size::Int=3) where {T <: Number} res_size = first(dims) reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) @@ -169,30 +218,44 @@ function cycle_jumps(rng::AbstractRNG, end """ - simple_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; - weight = T(0.1)) where {T <: Number} + simple_cycle([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + weight = 0.1) Create a simple cycle reservoir with the specified dimensions and weight. # Arguments - - `rng::AbstractRNG`: Random number generator. - - `T::Type`: Type of the elements in the reservoir matrix. - - `dims::Integer...`: Dimensions of the reservoir matrix. - - `weight::T = T(0.1)`: Weight of the connections in the reservoir matrix. - -# Returns - -Reservoir matrix with the dimensions specified by `dims` and weights. - -# References + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + - `weight`: Weight of the connections in the reservoir matrix. + Default is 0.1. + +# Examples + +```jldoctest +julia> res_matrix = simple_cycle(5, 5) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 0.1 + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.0 0.0 0.0 + 0.0 0.0 0.1 0.0 0.0 + 0.0 0.0 0.0 0.1 0.0 + +julia> res_matrix = simple_cycle(5, 5; weight=11) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 11.0 + 11.0 0.0 0.0 0.0 0.0 + 0.0 11.0 0.0 0.0 0.0 + 0.0 0.0 11.0 0.0 0.0 + 0.0 0.0 0.0 11.0 0.0 +``` [^Rodan2010]: Rodan, Ali, and Peter Tino. "Minimum complexity echo state network." IEEE transactions on neural networks 22.1 (2010): 131-144. """ -function simple_cycle(rng::AbstractRNG, - ::Type{T}, - dims::Integer...; +function simple_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; weight=T(0.1)) where {T <: Number} reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) @@ -205,37 +268,44 @@ function simple_cycle(rng::AbstractRNG, end """ - pseudo_svd(rng::AbstractRNG, ::Type{T}, dims::Integer...; - max_value, sparsity, sorted = true, reverse_sort = false) where {T <: Number} + pseudo_svd([rng::AbstractRNG=Utils.default_rng()], [T=Float32], dims...; + max_value=1.0, sparsity=0.1, sorted = true, reverse_sort = false) - Returns an initializer to build a sparse reservoir matrix with the given `sparsity` by using a pseudo-SVD approach as described in [^yang]. +Returns an initializer to build a sparse reservoir matrix with the given +`sparsity` by using a pseudo-SVD approach as described in [^yang]. # Arguments - - `rng::AbstractRNG`: Random number generator. - - `T::Type`: Type of the elements in the reservoir matrix. - - `dims::Integer...`: Dimensions of the reservoir matrix. + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. + Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. - `max_value`: The maximum absolute value of elements in the matrix. + Default is 1.0 - `sparsity`: The desired sparsity level of the reservoir matrix. - - `sorted`: A boolean indicating whether to sort the singular values before creating the diagonal matrix. By default, it is set to `true`. - - `reverse_sort`: A boolean indicating whether to reverse the sorted singular values. By default, it is set to `false`. - -# Returns - -Reservoir matrix with the specified dimensions, max value, and sparsity. - -# References - -This reservoir initialization method, based on a pseudo-SVD approach, is inspired by the work in [^yang], which focuses on designing polynomial echo state networks for time series prediction. + Default is 0.1 + - `sorted`: A boolean indicating whether to sort the singular values before + creating the diagonal matrix. Default is `true`. + - `reverse_sort`: A boolean indicating whether to reverse the sorted + singular values. Default is `false`. + +# Examples + +```jldoctest +julia> res_matrix = pseudo_svd(5, 5) +5×5 Matrix{Float32}: + 0.306998 0.0 0.0 0.0 0.0 + 0.0 0.325977 0.0 0.0 0.0 + 0.0 0.0 0.549051 0.0 0.0 + 0.0 0.0 0.0 0.726199 0.0 + 0.0 0.0 0.0 0.0 1.0 +``` [^yang]: Yang, Cuili, et al. "_Design of polynomial echo state networks for time series prediction._" Neurocomputing 290 (2018): 148-160. """ -function pseudo_svd(rng::AbstractRNG, - ::Type{T}, - dims::Integer...; - max_value::Number=T(1.0), - sparsity::Number=0.1, - sorted::Bool=true, +function pseudo_svd(rng::AbstractRNG, ::Type{T}, dims::Integer...; + max_value::Number=T(1.0), sparsity::Number=0.1, sorted::Bool=true, reverse_sort::Bool=false) where {T <: Number} reservoir_matrix = create_diag(rng, T, dims[1], max_value; @@ -283,9 +353,7 @@ function create_diag(rng::AbstractRNG, ::Type{T}, dim::Number, max_value::Number end function create_qmatrix(rng::AbstractRNG, ::Type{T}, dim::Number, - coord_i::Number, - coord_j::Number, - theta::Number) where {T <: Number} + coord_i::Number, coord_j::Number, theta::Number) where {T <: Number} qmatrix = DeviceAgnostic.zeros(rng, T, dim, dim) for i in 1:dim