fixed initializers and removed Distributions from dependencies

SciML · Dec 30, 2024 · 5872192 · 5872192
1 parent 4b81176
commit 5872192
Show file tree

Hide file tree

Showing 65 changed files with 55 additions and 47 deletions.
diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml
diff --git a/.buildkite/documentation.yml b/.buildkite/documentation.yml
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml
diff --git a/.github/workflows/Downgrade.yml b/.github/workflows/Downgrade.yml
diff --git a/.github/workflows/FormatCheck.yml b/.github/workflows/FormatCheck.yml
diff --git a/.github/workflows/Invalidations.yml b/.github/workflows/Invalidations.yml
diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml
diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml
diff --git a/.gitignore b/.gitignore
diff --git a/.typos.toml b/.typos.toml
diff --git a/CITATION.bib b/CITATION.bib
diff --git a/LICENSE b/LICENSE
diff --git a/Project.toml b/Project.toml
@@ -9,8 +9,10 @@ CellularAutomata = "878138dc-5b27-11ea-1a71-cb95d38d6b29"
 Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+Optim = "429524aa-4258-5aef-a3af-852621145aeb"
 PartialFunctions = "570af359-4316-4cb7-8c74-252c00c2016b"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 WeightInitializers = "d49dbf32-c5c2-4618-8acc-27bb2598ef2d"
 
@@ -32,8 +34,10 @@ LIBSVM = "0.8"
 LinearAlgebra = "1.10"
 MLJLinearModels = "0.9.2, 0.10"
 NNlib = "0.8.4, 0.9"
+Optim = "1"
 PartialFunctions = "1.2"
 Random = "1.10"
+Reexport = "1.2.2"
 SafeTestsets = "0.1"
 Statistics = "1.10"
 Test = "1"

diff --git a/README.md b/README.md
@@ -1,25 +1,19 @@
 <p align="center">
     <img width="400px" src="docs/src/assets/logo.png"/>
 </p>
-
 <div align="center">
-
 [![Join the chat at https://julialang.zulipchat.com #sciml-bridged](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/279055-sciml-bridged)
 [![Global Docs](https://img.shields.io/badge/docs-SciML-blue.svg)](https://docs.sciml.ai/ReservoirComputing/stable/)
 [![arXiv](https://img.shields.io/badge/arXiv-2204.05117-00b300.svg)](https://arxiv.org/abs/2204.05117)
-
 [![codecov](https://codecov.io/gh/SciML/ReservoirComputing.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/SciML/ReservoirComputing.jl)
 [![Build Status](https://github.com/SciML/ReservoirComputing.jl/workflows/CI/badge.svg)](https://github.com/SciML/ReservoirComputing.jl/actions?query=workflow%3ACI)
 [![Build status](https://badge.buildkite.com/db8f91b89a10ad79bbd1d9fdb1340e6f6602a1c0ed9496d4d0.svg)](https://buildkite.com/julialang/reservoircomputing-dot-jl)
-
 [![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor%27s%20Guide-blueviolet)](https://github.com/SciML/ColPrac)
 [![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle)
 
 </div>
-
 # ReservoirComputing.jl
 ReservoirComputing.jl provides an efficient, modular and easy to use implementation of Reservoir Computing models such as Echo State Networks (ESNs). For information on using this package please refer to the [stable documentation](https://docs.sciml.ai/ReservoirComputing/stable/). Use the [in-development documentation](https://docs.sciml.ai/ReservoirComputing/dev/) to take a look at at not yet released features.
-
 ## Quick Example
 
 To illustrate the workflow of this library we will showcase how it is possible to train an ESN to learn the dynamics of the Lorenz system. As a first step we will need to gather the data. For the `Generative` prediction we need the target data to be one step ahead of the training data:

diff --git a/docs/Project.toml b/docs/Project.toml
diff --git a/docs/make.jl b/docs/make.jl
diff --git a/docs/pages.jl b/docs/pages.jl
diff --git a/docs/src/api/esn.md b/docs/src/api/esn.md
diff --git a/docs/src/api/esn_drivers.md b/docs/src/api/esn_drivers.md
diff --git a/docs/src/api/predict.md b/docs/src/api/predict.md
diff --git a/docs/src/api/reca.md b/docs/src/api/reca.md
diff --git a/docs/src/api/states.md b/docs/src/api/states.md
diff --git a/docs/src/api/training.md b/docs/src/api/training.md
diff --git a/docs/src/assets/favicon.ico b/docs/src/assets/favicon.ico
diff --git a/docs/src/assets/logo.png b/docs/src/assets/logo.png
diff --git a/docs/src/esn_tutorials/change_layers.md b/docs/src/esn_tutorials/change_layers.md
diff --git a/docs/src/esn_tutorials/data/santafe_laser.txt b/docs/src/esn_tutorials/data/santafe_laser.txt
diff --git a/docs/src/esn_tutorials/deep_esn.md b/docs/src/esn_tutorials/deep_esn.md
diff --git a/docs/src/esn_tutorials/different_drivers.md b/docs/src/esn_tutorials/different_drivers.md
diff --git a/docs/src/esn_tutorials/different_training.md b/docs/src/esn_tutorials/different_training.md
diff --git a/docs/src/esn_tutorials/hybrid.md b/docs/src/esn_tutorials/hybrid.md
diff --git a/docs/src/esn_tutorials/lorenz_basic.md b/docs/src/esn_tutorials/lorenz_basic.md
diff --git a/docs/src/general/different_training.md b/docs/src/general/different_training.md
diff --git a/docs/src/general/predictive_generative.md b/docs/src/general/predictive_generative.md
diff --git a/docs/src/general/states_variation.md b/docs/src/general/states_variation.md
diff --git a/docs/src/index.md b/docs/src/index.md
diff --git a/docs/src/reca_tutorials/5bitinput.txt b/docs/src/reca_tutorials/5bitinput.txt
diff --git a/docs/src/reca_tutorials/5bitoutput.txt b/docs/src/reca_tutorials/5bitoutput.txt
diff --git a/docs/src/reca_tutorials/reca.md b/docs/src/reca_tutorials/reca.md
diff --git a/ext/RCLIBSVMExt.jl b/ext/RCLIBSVMExt.jl
diff --git a/ext/RCMLJLinearModelsExt.jl b/ext/RCMLJLinearModelsExt.jl
diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl
@@ -3,14 +3,15 @@ module ReservoirComputing
 using Adapt
 using CellularAutomata
 using Distances
-using Distributions
 using LinearAlgebra
 using NNlib
 using Optim
 using PartialFunctions
 using Random
+using Reexport: Reexport, @reexport
 using Statistics
-using WeightInitializers: WeightInitializers, DeviceAgnostic
+@reexport using WeightInitializers: WeightInitializers, DeviceAgnostic, PartialFunction,
+                                    Utils, sparse_init
 
 export NLADefault, NLAT1, NLAT2, NLAT3
 export StandardStates, ExtendedStates, PaddedStates, PaddedExtendedStates

diff --git a/src/esn/deepesn.jl b/src/esn/deepesn.jl
diff --git a/src/esn/esn.jl b/src/esn/esn.jl
diff --git a/src/esn/esn_input_layers.jl b/src/esn/esn_input_layers.jl
@@ -26,7 +26,8 @@ function scaled_rand(rng::AbstractRNG,
         dims::Integer...;
         scaling = T(0.1)) where {T <: Number}
     res_size, in_size = dims
-    layer_matrix = T.(DeviceAgnostic.rand(rng, Uniform(-scaling, scaling), res_size, in_size))
+    layer_matrix = (DeviceAgnostic.rand(rng, T, res_size, in_size) .- T(0.5)) .*
+                   (T(2) * scaling)
     return layer_matrix
 end
 
@@ -65,13 +66,12 @@ function weighted_init(rng::AbstractRNG,
         scaling = T(0.1)) where {T <: Number}
     approx_res_size, in_size = dims
     res_size = Int(floor(approx_res_size / in_size) * in_size)
-    layer_matrix = DeviceAgnostic.zeros(T, res_size, in_size)
+    layer_matrix = DeviceAgnostic.zeros(rng, T, res_size, in_size)
     q = floor(Int, res_size / in_size)
 
     for i in 1:in_size
-        layer_matrix[((i - 1) * q + 1):((i) * q), i] = DeviceAgnostic.rand(rng,
-            Uniform(-scaling, scaling),
-            q)
+        layer_matrix[((i - 1) * q + 1):((i) * q), i] = (DeviceAgnostic.rand(rng, T, q) .-
+                                                        T(0.5)) .* (T(2) * scaling)
     end
 
     return layer_matrix
@@ -113,25 +113,28 @@ function informed_init(rng::AbstractRNG, ::Type{T}, dims::Integer...;
         throw(DimensionMismatch("in_size must be greater than model_in_size"))
     end
 
-    input_matrix = DeviceAgnostic.zeros(res_size, in_size)
-    zero_connections = DeviceAgnostic.zeros(in_size)
+    input_matrix = DeviceAgnostic.zeros(rng, T, res_size, in_size)
+    zero_connections = DeviceAgnostic.zeros(rng, T, in_size)
     num_for_state = floor(Int, res_size * gamma)
     num_for_model = floor(Int, res_size * (1 - gamma))
 
     for i in 1:num_for_state
         idxs = findall(Bool[zero_connections .== input_matrix[i, :]
                             for i in 1:size(input_matrix, 1)])
-        random_row_idx = idxs[DeviceAgnostic.rand(rng, 1:end)]
-        random_clm_idx = range(1, state_size, step = 1)[DeviceAgnostic.rand(rng, 1:end)]
-        input_matrix[random_row_idx, random_clm_idx] = DeviceAgnostic.rand(rng, Uniform(-scaling, scaling))
+        random_row_idx = idxs[DeviceAgnostic.rand(rng, T, 1:end)]
+        random_clm_idx = range(1, state_size, step = 1)[DeviceAgnostic.rand(rng, T, 1:end)]
+        input_matrix[random_row_idx, random_clm_idx] = (DeviceAgnostic.rand(rng, T) -
+                                                        T(0.5)) .* (T(2) * scaling)
     end
 
     for i in 1:num_for_model
         idxs = findall(Bool[zero_connections .== input_matrix[i, :]
                             for i in 1:size(input_matrix, 1)])
-        random_row_idx = idxs[DeviceAgnostic.rand(rng, 1:end)]
-        random_clm_idx = range(state_size + 1, in_size, step = 1)[DeviceAgnostic.rand(rng, 1:end)]
-        input_matrix[random_row_idx, random_clm_idx] = DeviceAgnostic.rand(rng, Uniform(-scaling, scaling))
+        random_row_idx = idxs[DeviceAgnostic.rand(rng, T, 1:end)]
+        random_clm_idx = range(state_size + 1, in_size, step = 1)[DeviceAgnostic.rand(
+            rng, T, 1:end)]
+        input_matrix[random_row_idx, random_clm_idx] = (DeviceAgnostic.rand(rng, T) -
+                                                        T(0.5)) .* (T(2) * scaling)
     end
 
     return input_matrix
@@ -196,11 +199,14 @@ function _create_bernoulli(p::Number,
         weight::Number,
         rng::AbstractRNG,
         ::Type{T}) where {T <: Number}
-    input_matrix = DeviceAgnostic.zeros(T, res_size, in_size)
+    input_matrix = DeviceAgnostic.zeros(rng, T, res_size, in_size)
     for i in 1:res_size
         for j in 1:in_size
-            DeviceAgnostic.rand(rng, Bernoulli(p)) ? (input_matrix[i, j] = weight) :
-            (input_matrix[i, j] = -weight)
+            if DeviceAgnostic.rand(rng, T) < p
+                input_matrix[i, j] = weight
+            else
+                input_matrix[i, j] = -weight
+            end
         end
     end
     return input_matrix
@@ -216,8 +222,8 @@ function _create_irrational(irrational::Irrational,
     setprecision(BigFloat, Int(ceil(log2(10) * (res_size * in_size + start + 1))))
     ir_string = string(BigFloat(irrational)) |> collect
     deleteat!(ir_string, findall(x -> x == '.', ir_string))
-    ir_array = DeviceAgnostic.zeros(length(ir_string))
-    input_matrix = DeviceAgnostic.zeros(T, res_size, in_size)
+    ir_array = DeviceAgnostic.zeros(rng, T, length(ir_string))
+    input_matrix = DeviceAgnostic.zeros(rng, T, res_size, in_size)
 
     for i in 1:length(ir_string)
         ir_array[i] = parse(Int, ir_string[i])

diff --git a/src/esn/esn_predict.jl b/src/esn/esn_predict.jl
diff --git a/src/esn/esn_reservoir_drivers.jl b/src/esn/esn_reservoir_drivers.jl
diff --git a/src/esn/esn_reservoirs.jl b/src/esn/esn_reservoirs.jl
@@ -66,7 +66,7 @@ function delay_line(rng::AbstractRNG,
         ::Type{T},
         dims::Integer...;
         weight = T(0.1)) where {T <: Number}
-    reservoir_matrix = DeviceAgnostic.zeros(T, dims...)
+    reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...)
     @assert length(dims) == 2&&dims[1] == dims[2] "The dimensions must define a square matrix (e.g., (100, 100))"
 
     for i in 1:(dims[1] - 1)
@@ -107,7 +107,7 @@ function delay_line_backward(rng::AbstractRNG,
         weight = T(0.1),
         fb_weight = T(0.2)) where {T <: Number}
     res_size = first(dims)
-    reservoir_matrix = DeviceAgnostic.zeros(T, dims...)
+    reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...)
 
     for i in 1:(res_size - 1)
         reservoir_matrix[i + 1, i] = weight
@@ -148,7 +148,7 @@ function cycle_jumps(rng::AbstractRNG,
         jump_weight::Number = T(0.1),
         jump_size::Int = 3) where {T <: Number}
     res_size = first(dims)
-    reservoir_matrix = DeviceAgnostic.zeros(T, dims...)
+    reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...)
 
     for i in 1:(res_size - 1)
         reservoir_matrix[i + 1, i] = cycle_weight
@@ -194,7 +194,7 @@ function simple_cycle(rng::AbstractRNG,
         ::Type{T},
         dims::Integer...;
         weight = T(0.1)) where {T <: Number}
-    reservoir_matrix = DeviceAgnostic.zeros(T, dims...)
+    reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...)
 
     for i in 1:(dims[1] - 1)
         reservoir_matrix[i + 1, i] = weight
@@ -237,38 +237,42 @@ function pseudo_svd(rng::AbstractRNG,
         sparsity::Number = 0.1,
         sorted::Bool = true,
         reverse_sort::Bool = false) where {T <: Number}
-    reservoir_matrix = create_diag(dims[1],
-        max_value,
-        T;
+    reservoir_matrix = create_diag(rng, T, dims[1],
+        max_value;
         sorted = sorted,
         reverse_sort = reverse_sort)
     tmp_sparsity = get_sparsity(reservoir_matrix, dims[1])
 
     while tmp_sparsity <= sparsity
-        reservoir_matrix *= create_qmatrix(dims[1],
-            DeviceAgnostic.rand(1:dims[1]),
-            DeviceAgnostic.rand(1:dims[1]),
-            DeviceAgnostic.rand(T) * T(2) - T(1),
-            T)
+        reservoir_matrix *= create_qmatrix(rng, T, dims[1],
+            rand_range(rng, T, dims[1]),
+            rand_range(rng, T, dims[1]),
+            DeviceAgnostic.rand(rng, T) * T(2) - T(1))
         tmp_sparsity = get_sparsity(reservoir_matrix, dims[1])
     end
 
     return reservoir_matrix
 end
 
-function create_diag(dim::Number, max_value::Number, ::Type{T};
+#hacky workaround for the moment
+function rand_range(rng, T, n::Int)
+    return Int(1 + floor(DeviceAgnostic.rand(rng, T) * n))
+end
+
+function create_diag(rng::AbstractRNG, ::Type{T}, dim::Number, max_value::Number;
         sorted::Bool = true, reverse_sort::Bool = false) where {T <: Number}
-    diagonal_matrix = DeviceAgnostic.zeros(T, dim, dim)
+    diagonal_matrix = DeviceAgnostic.zeros(rng, T, dim, dim)
     if sorted == true
         if reverse_sort == true
-            diagonal_values = sort(DeviceAgnostic.rand(T, dim) .* max_value, rev = true)
+            diagonal_values = sort(
+                DeviceAgnostic.rand(rng, T, dim) .* max_value, rev = true)
             diagonal_values[1] = max_value
         else
-            diagonal_values = sort(DeviceAgnostic.rand(T, dim) .* max_value)
+            diagonal_values = sort(DeviceAgnostic.rand(rng, T, dim) .* max_value)
             diagonal_values[end] = max_value
         end
     else
-        diagonal_values = DeviceAgnostic.rand(T, dim) .* max_value
+        diagonal_values = DeviceAgnostic.rand(rng, T, dim) .* max_value
     end
 
     for i in 1:dim
@@ -278,12 +282,11 @@ function create_diag(dim::Number, max_value::Number, ::Type{T};
     return diagonal_matrix
 end
 
-function create_qmatrix(dim::Number,
+function create_qmatrix(rng::AbstractRNG, ::Type{T}, dim::Number,
         coord_i::Number,
         coord_j::Number,
-        theta::Number,
-        ::Type{T}) where {T <: Number}
-    qmatrix = DeviceAgnostic.zeros(T, dim, dim)
+        theta::Number) where {T <: Number}
+    qmatrix = DeviceAgnostic.zeros(rng, T, dim, dim)
 
     for i in 1:dim
         qmatrix[i, i] = 1.0

diff --git a/src/esn/hybridesn.jl b/src/esn/hybridesn.jl
diff --git a/src/predict.jl b/src/predict.jl
diff --git a/src/reca/reca.jl b/src/reca/reca.jl
diff --git a/src/reca/reca_input_encodings.jl b/src/reca/reca_input_encodings.jl
diff --git a/src/states.jl b/src/states.jl
diff --git a/src/train/linear_regression.jl b/src/train/linear_regression.jl
diff --git a/test/esn/deepesn.jl b/test/esn/deepesn.jl
diff --git a/test/esn/test_drivers.jl b/test/esn/test_drivers.jl
diff --git a/test/esn/test_hybrid.jl b/test/esn/test_hybrid.jl
diff --git a/test/esn/test_inits.jl b/test/esn/test_inits.jl
diff --git a/test/esn/test_train.jl b/test/esn/test_train.jl
diff --git a/test/qa.jl b/test/qa.jl
diff --git a/test/reca/test_predictive.jl b/test/reca/test_predictive.jl
diff --git a/test/runtests.jl b/test/runtests.jl
diff --git a/test/test_states.jl b/test/test_states.jl