From 557f3d9e2c3074649ac7a26a56a283b87cc88f2c Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 23 Nov 2022 14:32:32 +0100
Subject: [PATCH 001/375] Initial commit

---
 lib/SimpleNonlinearSolve/README.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 lib/SimpleNonlinearSolve/README.md

diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
new file mode 100644
index 000000000..ef116ffa8
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -0,0 +1 @@
+# SimpleNonlinearSolve

From 046c7173d5f766812719a38245fda8081f1fd2e3 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 23 Nov 2022 14:33:12 +0100
Subject: [PATCH 002/375] really start repo

---
 lib/SimpleNonlinearSolve/.JuliaFormatter.toml |   1 +
 lib/SimpleNonlinearSolve/LICENSE              |  21 +++
 lib/SimpleNonlinearSolve/Project.toml         |  38 +++++
 lib/SimpleNonlinearSolve/README.md            |  39 ++++-
 .../src/SimpleNonlinearSolve.jl               |  29 ++++
 lib/SimpleNonlinearSolve/src/ad.jl            |  63 ++++++++
 lib/SimpleNonlinearSolve/src/bisection.jl     |  65 ++++++++
 lib/SimpleNonlinearSolve/src/falsi.jl         |  68 ++++++++
 lib/SimpleNonlinearSolve/src/raphson.jl       |  51 ++++++
 lib/SimpleNonlinearSolve/src/utils.jl         |  35 ++++
 lib/SimpleNonlinearSolve/test/basictests.jl   | 152 ++++++++++++++++++
 lib/SimpleNonlinearSolve/test/runtests.jl     |  12 ++
 lib/SimpleNonlinearSolve/workflows/CI.yml     |  42 +++++
 .../workflows/CompatHelper.yml                |  26 +++
 .../workflows/Documentation.yml               |  29 ++++
 .../workflows/Downstream.yml                  |  53 ++++++
 .../workflows/FormatCheck.yml                 |  42 +++++
 .../workflows/Invalidations.yml               |  40 +++++
 lib/SimpleNonlinearSolve/workflows/TagBot.yml |  15 ++
 19 files changed, 820 insertions(+), 1 deletion(-)
 create mode 100644 lib/SimpleNonlinearSolve/.JuliaFormatter.toml
 create mode 100644 lib/SimpleNonlinearSolve/LICENSE
 create mode 100644 lib/SimpleNonlinearSolve/Project.toml
 create mode 100644 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/ad.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/bisection.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/falsi.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/raphson.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/utils.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/basictests.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/runtests.jl
 create mode 100644 lib/SimpleNonlinearSolve/workflows/CI.yml
 create mode 100644 lib/SimpleNonlinearSolve/workflows/CompatHelper.yml
 create mode 100644 lib/SimpleNonlinearSolve/workflows/Documentation.yml
 create mode 100644 lib/SimpleNonlinearSolve/workflows/Downstream.yml
 create mode 100644 lib/SimpleNonlinearSolve/workflows/FormatCheck.yml
 create mode 100644 lib/SimpleNonlinearSolve/workflows/Invalidations.yml
 create mode 100644 lib/SimpleNonlinearSolve/workflows/TagBot.yml

diff --git a/lib/SimpleNonlinearSolve/.JuliaFormatter.toml b/lib/SimpleNonlinearSolve/.JuliaFormatter.toml
new file mode 100644
index 000000000..453925c3f
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/.JuliaFormatter.toml
@@ -0,0 +1 @@
+style = "sciml"
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/LICENSE b/lib/SimpleNonlinearSolve/LICENSE
new file mode 100644
index 000000000..4d2bf6e69
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Julia Computing, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
new file mode 100644
index 000000000..0e3a98cd2
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -0,0 +1,38 @@
+name = "SimpleNonlinearSolve"
+uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
+authors = ["Kanav Gupta <kanav0610@gmail.com>"]
+version = "0.1.0"
+
+[deps]
+ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
+FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
+SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
+
+[compat]
+ArrayInterfaceCore = "0.1.1"
+FiniteDiff = "2"
+ForwardDiff = "0.10.3"
+RecursiveArrayTools = "2"
+Reexport = "0.2, 1"
+SciMLBase = "1.32"
+Setfield = "0.7, 0.8, 1"
+StaticArrays = "0.12,1.0"
+UnPack = "1.0"
+julia = "1.6"
+
+[extras]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "ForwardDiff"]
diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index ef116ffa8..ef305c0c8 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -1 +1,38 @@
-# SimpleNonlinearSolve
+# NonlinearSolve.jl
+
+[![Join the chat at https://julialang.zulipchat.com #sciml-bridged](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/279055-sciml-bridged)
+[![Global Docs](https://img.shields.io/badge/docs-SciML-blue.svg)](https://docs.sciml.ai/NonlinearSolve/stable/)
+
+[![codecov](https://codecov.io/gh/SciML/NonlinearSolve.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/SciML/NonlinearSolve.jl)
+[![Build Status](https://github.com/SciML/NonlinearSolve.jl/workflows/CI/badge.svg)](https://github.com/SciML/NonlinearSolve.jl/actions?query=workflow%3ACI)
+
+[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac)
+[![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle)
+
+
+
+
+Fast implementations of root finding algorithms in Julia that satisfy the SciML common interface.
+
+For information on using the package,
+[see the stable documentation](https://docs.sciml.ai/NonlinearSolve/stable/). Use the
+[in-development documentation](https://docs.sciml.ai/NonlinearSolve/dev/) for the version of
+the documentation which contains the unreleased features.
+
+## High Level Examples
+
+```julia
+using NonlinearSolve, StaticArrays
+
+f(u,p) = u .* u .- 2
+u0 = @SVector[1.0, 1.0]
+probN = NonlinearProblem{false}(f, u0)
+solver = solve(probN, NewtonRaphson(), tol = 1e-9)
+
+## Bracketing Methods
+
+f(u, p) = u .* u .- 2.0
+u0 = (1.0, 2.0) # brackets
+probB = NonlinearProblem(f, u0)
+sol = solve(probB, Falsi())
+```
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
new file mode 100644
index 000000000..191bb8731
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -0,0 +1,29 @@
+module SimpleNonlinearSolve
+
+using Reexport
+using UnPack: @unpack
+using FiniteDiff, ForwardDiff
+using ForwardDiff: Dual
+using Setfield
+using StaticArrays
+using RecursiveArrayTools
+using LinearAlgebra
+import ArrayInterfaceCore
+
+@reexport using SciMLBase
+
+abstract type AbstractSimpleNonlinearSolveAlgorithm <: SciMLBase.AbstractNonlinearAlgorithm end
+abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
+abstract type AbstractNewtonAlgorithm{CS, AD, FDT} <: AbstractSimpleNonlinearSolveAlgorithm end
+abstract type AbstractImmutableNonlinearSolver <: AbstractSimpleNonlinearSolveAlgorithm end
+
+include("utils.jl")
+include("bisection.jl")
+include("falsi.jl")
+include("raphson.jl")
+include("ad.jl")
+
+# DiffEq styled algorithms
+export Bisection, Falsi, SimpleNewtonRaphson
+
+end # module
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
new file mode 100644
index 000000000..07a0620d0
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -0,0 +1,63 @@
+function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
+    f = prob.f
+    p = value(prob.p)
+    u0 = value(prob.u0)
+
+    newprob = NonlinearProblem(f, u0, p; prob.kwargs...)
+    sol = solve(newprob, alg, args...; kwargs...)
+
+    uu = sol.u
+    if p isa Number
+        f_p = ForwardDiff.derivative(Base.Fix1(f, uu), p)
+    else
+        f_p = ForwardDiff.gradient(Base.Fix1(f, uu), p)
+    end
+
+    f_x = ForwardDiff.derivative(Base.Fix2(f, p), uu)
+    pp = prob.p
+    sumfun = let f_x′ = -f_x
+        ((fp, p),) -> (fp / f_x′) * ForwardDiff.partials(p)
+    end
+    partials = sum(sumfun, zip(f_p, pp))
+    return sol, partials
+end
+
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, SVector}, iip,
+                                                <:Dual{T, V, P}}, alg::SimpleNewtonRaphson,
+                         args...; kwargs...) where {iip, T, V, P}
+    sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
+    return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
+                                    retcode = sol.retcode)
+end
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, SVector}, iip,
+                                                <:AbstractArray{<:Dual{T, V, P}}},
+                         alg::SimpleNewtonRaphson, args...; kwargs...) where {iip, T, V, P}
+    sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
+    return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
+                                    retcode = sol.retcode)
+end
+
+# avoid ambiguities
+for Alg in [Bisection]
+    @eval function SciMLBase.solve(prob::NonlinearProblem{uType, iip, <:Dual{T, V, P}},
+                                   alg::$Alg, args...;
+                                   kwargs...) where {uType, iip, T, V, P}
+        sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
+        return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials),
+                                        sol.resid; retcode = sol.retcode,
+                                        left = Dual{T, V, P}(sol.left, partials),
+                                        right = Dual{T, V, P}(sol.right, partials))
+        #return BracketingSolution(Dual{T,V,P}(sol.left, partials), Dual{T,V,P}(sol.right, partials), sol.retcode, sol.resid)
+    end
+    @eval function SciMLBase.solve(prob::NonlinearProblem{uType, iip,
+                                                          <:AbstractArray{<:Dual{T, V, P}}},
+                                   alg::$Alg, args...;
+                                   kwargs...) where {uType, iip, T, V, P}
+        sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
+        return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials),
+                                        sol.resid; retcode = sol.retcode,
+                                        left = Dual{T, V, P}(sol.left, partials),
+                                        right = Dual{T, V, P}(sol.right, partials))
+        #return BracketingSolution(Dual{T,V,P}(sol.left, partials), Dual{T,V,P}(sol.right, partials), sol.retcode, sol.resid)
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
new file mode 100644
index 000000000..6e00afba7
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -0,0 +1,65 @@
+struct Bisection <: AbstractBracketingAlgorithm
+    exact_left::Bool
+    exact_right::Bool
+end
+
+function Bisection(; exact_left = false, exact_right = false)
+    Bisection(exact_left, exact_right)
+end
+
+function SciMLBase.solve(prob::NonlinearProblem, alg::Bisection, args...; maxiters = 1000,
+                         kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    left, right = prob.u0
+    fl, fr = f(left), f(right)
+
+    if iszero(fl)
+        return SciMLBase.build_solution(prob, alg, left, fl;
+                                        retcode = ReturnCode.ExactSolutionLeft, left = left,
+                                        right = right)
+    end
+
+    i = 1
+    if !iszero(fr)
+        while i < maxiters
+            mid = (left + right) / 2
+            (mid == left || mid == right) &&
+                return SciMLBase.build_solution(prob, alg, left, fl;
+                                                retcode = ReturnCode.FloatingPointLimit,
+                                                left = left, right = right)
+            fm = f(mid)
+            if iszero(fm)
+                right = mid
+                break
+            end
+            if sign(fl) == sign(fm)
+                fl = fm
+                left = mid
+            else
+                fr = fm
+                right = mid
+            end
+            i += 1
+        end
+    end
+
+    while i < maxiters
+        mid = (left + right) / 2
+        (mid == left || mid == right) &&
+            return SciMLBase.build_solution(prob, alg, left, fl;
+                                            retcode = ReturnCode.FloatingPointLimit,
+                                            left = left, right = right)
+        fm = f(mid)
+        if iszero(fm)
+            right = mid
+            fr = fm
+        else
+            left = mid
+            fl = fm
+        end
+        i += 1
+    end
+
+    return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
+                                    left = left, right = right)
+end
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
new file mode 100644
index 000000000..0536ece19
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -0,0 +1,68 @@
+struct Falsi <: AbstractBracketingAlgorithm end
+
+function SciMLBase.solve(prob::NonlinearProblem, alg::Falsi, args...; maxiters = 1000,
+                         kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    left, right = prob.u0
+    fl, fr = f(left), f(right)
+
+    if iszero(fl)
+        return SciMLBase.build_solution(prob, alg, left, fl;
+                                        retcode = ReturnCode.ExactSolutionLeft, left = left,
+                                        right = right)
+    end
+
+    i = 1
+    if !iszero(fr)
+        while i < maxiters
+            if nextfloat_tdir(left, prob.u0...) == right
+                return SciMLBase.build_solution(prob, alg, left, fl;
+                                                retcode = ReturnCode.FloatingPointLimit,
+                                                left = left, right = right)
+            end
+            mid = (fr * left - fl * right) / (fr - fl)
+            for i in 1:10
+                mid = max_tdir(left, prevfloat_tdir(mid, prob.u0...), prob.u0...)
+            end
+            if mid == right || mid == left
+                break
+            end
+            fm = f(mid)
+            if iszero(fm)
+                right = mid
+                break
+            end
+            if sign(fl) == sign(fm)
+                fl = fm
+                left = mid
+            else
+                fr = fm
+                right = mid
+            end
+            i += 1
+        end
+    end
+
+    while i < maxiters
+        mid = (left + right) / 2
+        (mid == left || mid == right) &&
+            return SciMLBase.build_solution(prob, alg, left, fl;
+                                            retcode = ReturnCode.FloatingPointLimit,
+                                            left = left, right = right)
+        fm = f(mid)
+        if iszero(fm)
+            right = mid
+            fr = fm
+        elseif sign(fm) == sign(fl)
+            left = mid
+            fl = fm
+        else
+            right = mid
+            fr = fm
+        end
+        i += 1
+    end
+
+    return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
+                                    left = left, right = right)
+end
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
new file mode 100644
index 000000000..8ed325a86
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -0,0 +1,51 @@
+struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
+    function SimpleNewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
+                                   diff_type = Val{:forward})
+        new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
+        SciMLBase._unwrap_val(diff_type)}()
+    end
+end
+
+function SciMLBase.solve(prob::NonlinearProblem,
+                         alg::SimpleNewtonRaphson, args...; xatol = nothing, xrtol = nothing,
+                         maxiters = 1000, kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    x = float(prob.u0)
+    fx = float(prob.u0)
+    T = typeof(x)
+
+    if SciMLBase.isinplace(prob)
+        error("SimpleNewtonRaphson currently only supports out-of-place nonlinear problems")
+    end
+
+    atol = xatol !== nothing ? xatol : oneunit(eltype(T)) * (eps(one(eltype(T))))^(4 // 5)
+    rtol = xrtol !== nothing ? xrtol : eps(one(eltype(T)))^(4 // 5)
+
+    if typeof(x) <: Number
+        xo = oftype(one(eltype(x)), Inf)
+    else
+        xo = map(x -> oftype(one(eltype(x)), Inf), x)
+    end
+
+    for i in 1:maxiters
+        if alg_autodiff(alg)
+            fx, dfx = value_derivative(f, x)
+        elseif x isa AbstractArray
+            fx = f(x)
+            dfx = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x), fx)
+        else
+            fx = f(x)
+            dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x),
+                                                          fx)
+        end
+        iszero(fx) &&
+            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Default)
+        Δx = dfx \ fx
+        x -= Δx
+        if isapprox(x, xo, atol = atol, rtol = rtol)
+            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Default)
+        end
+        xo = x
+    end
+    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
new file mode 100644
index 000000000..25f6ba358
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -0,0 +1,35 @@
+"""
+  prevfloat_tdir(x, x0, x1)
+
+Move `x` one floating point towards x0.
+"""
+function prevfloat_tdir(x, x0, x1)
+    x1 > x0 ? prevfloat(x) : nextfloat(x)
+end
+
+function nextfloat_tdir(x, x0, x1)
+    x1 > x0 ? nextfloat(x) : prevfloat(x)
+end
+
+function max_tdir(a, b, x0, x1)
+    x1 > x0 ? max(a, b) : min(a, b)
+end
+
+alg_autodiff(alg::AbstractNewtonAlgorithm{CS, AD, FDT}) where {CS, AD, FDT} = AD
+diff_type(alg::AbstractNewtonAlgorithm{CS, AD, FDT}) where {CS, AD, FDT} = FDT
+
+"""
+  value_derivative(f, x)
+
+Compute `f(x), d/dx f(x)` in the most efficient way.
+"""
+function value_derivative(f::F, x::R) where {F, R}
+    T = typeof(ForwardDiff.Tag(f, R))
+    out = f(ForwardDiff.Dual{T}(x, one(x)))
+    ForwardDiff.value(out), ForwardDiff.extract_derivative(T, out)
+end
+value_derivative(f::F, x::AbstractArray) where {F} = f(x), ForwardDiff.jacobian(f, x)
+
+value(x) = x
+value(x::Dual) = ForwardDiff.value(x)
+value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
new file mode 100644
index 000000000..47143dbce
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -0,0 +1,152 @@
+using SimpleNonlinearSolve
+using StaticArrays
+using BenchmarkTools
+using Test
+
+function benchmark_scalar(f, u0)
+    probN = NonlinearProblem{false}(f, u0)
+    sol = (solve(probN, SimpleNewtonRaphson()))
+end
+
+function ff(u, p)
+    u .* u .- 2
+end
+const cu0 = @SVector[1.0, 1.0]
+function sf(u, p)
+    u * u - 2
+end
+const csu0 = 1.0
+
+sol = benchmark_scalar(sf, csu0)
+@test sol.retcode === ReturnCode.Default
+@test sol.u * sol.u - 2 < 1e-9
+
+@test (@ballocated benchmark_scalar(sf, csu0)) == 0
+
+# AD Tests
+using ForwardDiff
+
+# Immutable
+f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
+
+g = function (p)
+    probN = NonlinearProblem{false}(f, csu0, p)
+    sol = solve(probN, SimpleNewtonRaphson(), tol = 1e-9)
+    return sol.u[end]
+end
+
+for p in 1.0:0.1:100.0
+    @test g(p) ≈ sqrt(p)
+    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+end
+
+# Scalar
+f, u0 = (u, p) -> u * u - p, 1.0
+
+# SimpleNewtonRaphson
+g = function (p)
+    probN = NonlinearProblem{false}(f, oftype(p, u0), p)
+    sol = solve(probN, SimpleNewtonRaphson())
+    return sol.u
+end
+
+@test ForwardDiff.derivative(g, 1.0) ≈ 0.5
+
+for p in 1.1:0.1:100.0
+    @test g(p) ≈ sqrt(p)
+    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+end
+
+u0 = (1.0, 20.0)
+# Falsi
+g = function (p)
+    probN = NonlinearProblem{false}(f, typeof(p).(u0), p)
+    sol = solve(probN, Falsi())
+    return sol.left
+end
+
+for p in 1.1:0.1:100.0
+    @test g(p) ≈ sqrt(p)
+    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+end
+
+f, u0 = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
+t = (p) -> [sqrt(p[2] / p[1])]
+p = [0.9, 50.0]
+for alg in [Bisection(), Falsi()]
+    global g, p
+    g = function (p)
+        probN = NonlinearProblem{false}(f, u0, p)
+        sol = solve(probN, Bisection())
+        return [sol.left]
+    end
+
+    @test g(p) ≈ [sqrt(p[2] / p[1])]
+    @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
+end
+
+gnewton = function (p)
+    probN = NonlinearProblem{false}(f, 0.5, p)
+    sol = solve(probN, SimpleNewtonRaphson())
+    return [sol.u]
+end
+@test gnewton(p) ≈ [sqrt(p[2] / p[1])]
+@test ForwardDiff.jacobian(gnewton, p) ≈ ForwardDiff.jacobian(t, p)
+
+# Error Checks
+
+f, u0 = (u, p) -> u .* u .- 2.0, @SVector[1.0, 1.0]
+probN = NonlinearProblem(f, u0)
+
+@test solve(probN, SimpleNewtonRaphson()).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleNewtonRaphson(); immutable = false).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
+
+for u0 in [1.0, [1, 1.0]]
+    local f, probN, sol
+    f = (u, p) -> u .* u .- 2.0
+    probN = NonlinearProblem(f, u0)
+    sol = sqrt(2) * u0
+
+    @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
+    @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
+    @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol
+end
+
+# Bisection Tests
+f, u0 = (u, p) -> u .* u .- 2.0, (1.0, 2.0)
+probB = NonlinearProblem(f, u0)
+
+# Falsi
+sol = solve(probB, Falsi())
+@test sol.left ≈ sqrt(2.0)
+
+sol = solve(probB, Bisection())
+@test sol.left ≈ sqrt(2.0)
+
+# Garuntee Tests for Bisection
+f = function (u, p)
+    if u < 2.0
+        return u - 2.0
+    elseif u > 3.0
+        return u - 3.0
+    else
+        return 0.0
+    end
+end
+probB = NonlinearProblem(f, (0.0, 4.0))
+
+sol = solve(probB, Bisection(; exact_left = true))
+@test f(sol.left, nothing) < 0.0
+@test f(nextfloat(sol.left), nothing) >= 0.0
+
+sol = solve(probB, Bisection(; exact_right = true))
+@test f(sol.right, nothing) >= 0.0
+@test f(prevfloat(sol.right), nothing) <= 0.0
+
+sol = solve(probB, Bisection(; exact_left = true, exact_right = true); immutable = false)
+@test f(sol.left, nothing) < 0.0
+@test f(nextfloat(sol.left), nothing) >= 0.0
+@test f(sol.right, nothing) >= 0.0
+@test f(prevfloat(sol.right), nothing) <= 0.0
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
new file mode 100644
index 000000000..7269ed6bf
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -0,0 +1,12 @@
+using Pkg
+using SafeTestsets
+const LONGER_TESTS = false
+
+const GROUP = get(ENV, "GROUP", "All")
+const is_APPVEYOR = Sys.iswindows() && haskey(ENV, "APPVEYOR")
+
+@time begin
+
+if GROUP == "All" || GROUP == "Core"
+    @time @safetestset "Basic Tests + Some AD" begin include("basictests.jl") end
+end end
diff --git a/lib/SimpleNonlinearSolve/workflows/CI.yml b/lib/SimpleNonlinearSolve/workflows/CI.yml
new file mode 100644
index 000000000..c8d041ecf
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/workflows/CI.yml
@@ -0,0 +1,42 @@
+name: CI
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        group:
+          - Core
+        version:
+          - '1'
+          - '1.6'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+      - uses: actions/cache@v1
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+        env:
+          GROUP: ${{ matrix.group }}
+          JULIA_NUM_THREADS: 11
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v1
+        with:
+          file: lcov.info
diff --git a/lib/SimpleNonlinearSolve/workflows/CompatHelper.yml b/lib/SimpleNonlinearSolve/workflows/CompatHelper.yml
new file mode 100644
index 000000000..73494545f
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/workflows/CompatHelper.yml
@@ -0,0 +1,26 @@
+name: CompatHelper
+
+on:
+  schedule:
+    - cron: '00 * * * *'
+  issues:
+    types: [opened, reopened]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        julia-version: [1]
+        julia-arch: [x86]
+        os: [ubuntu-latest]
+    steps:
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: ${{ matrix.julia-version }}
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: julia -e 'using CompatHelper; CompatHelper.main(;subdirs=["", "docs"])'
diff --git a/lib/SimpleNonlinearSolve/workflows/Documentation.yml b/lib/SimpleNonlinearSolve/workflows/Documentation.yml
new file mode 100644
index 000000000..f64a315b6
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/workflows/Documentation.yml
@@ -0,0 +1,29 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - master
+      - 'release-'
+    tags: '*'
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: '1'
+      - name: Install dependencies
+        run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
+      - name: Build and deploy
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For authentication with GitHub Actions token
+          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key
+        run: julia --project=docs/ --code-coverage=user docs/make.jl
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v1
+        with:
+          file: lcov.info
diff --git a/lib/SimpleNonlinearSolve/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/workflows/Downstream.yml
new file mode 100644
index 000000000..916a33b4b
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/workflows/Downstream.yml
@@ -0,0 +1,53 @@
+name: IntegrationTest
+on:
+  push:
+    branches: [master]
+    tags: [v*]
+  pull_request:
+
+jobs:
+  test:
+    name: ${{ matrix.package.repo }}/${{ matrix.package.group }}/${{ matrix.julia-version }}
+    runs-on: ${{ matrix.os }}
+    env:
+      GROUP: ${{ matrix.package.group }}
+    strategy:
+      fail-fast: false
+      matrix:
+        julia-version: [1,1.6]
+        os: [ubuntu-latest]
+        package:
+          - {user: SciML, repo: ModelingToolkit.jl, group: All}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.julia-version }}
+          arch: x64
+      - uses: julia-actions/julia-buildpkg@latest
+      - name: Clone Downstream
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ matrix.package.user }}/${{ matrix.package.repo }}
+          path: downstream
+      - name: Load this and run the downstream tests
+        shell: julia --color=yes --project=downstream {0}
+        run: |
+          using Pkg
+          try
+            # force it to use this PR's version of the package
+            Pkg.develop(PackageSpec(path="."))  # resolver may fail with main deps
+            Pkg.update()
+            Pkg.test(coverage=true)  # resolver may fail with test time deps
+          catch err
+            err isa Pkg.Resolve.ResolverError || rethrow()
+            # If we can't resolve that means this is incompatible by SemVer and this is fine
+            # It means we marked this as a breaking change, so we don't need to worry about
+            # Mistakenly introducing a breaking change, as we have intentionally made one
+            @info "Not compatible with this release. No problem." exception=err
+            exit(0)  # Exit immediately, as a success
+          end
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v1
+        with:
+          file: lcov.info
diff --git a/lib/SimpleNonlinearSolve/workflows/FormatCheck.yml b/lib/SimpleNonlinearSolve/workflows/FormatCheck.yml
new file mode 100644
index 000000000..2a3517a0f
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/workflows/FormatCheck.yml
@@ -0,0 +1,42 @@
+name: format-check
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'release-'
+    tags: '*'
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        julia-version: [1]
+        julia-arch: [x86]
+        os: [ubuntu-latest]
+    steps:
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: ${{ matrix.julia-version }}
+
+      - uses: actions/checkout@v1
+      - name: Install JuliaFormatter and format
+        # This will use the latest version by default but you can set the version like so:
+        #
+        # julia  -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter", version="0.13.0"))'
+        run: |
+          julia  -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter"))'
+          julia  -e 'using JuliaFormatter; format(".", verbose=true)'
+      - name: Format check
+        run: |
+          julia -e '
+          out = Cmd(`git diff --name-only`) |> read |> String
+          if out == ""
+              exit(0)
+          else
+              @error "Some files have not been formatted !!!"
+              write(stdout, out)
+              exit(1)
+          end'
diff --git a/lib/SimpleNonlinearSolve/workflows/Invalidations.yml b/lib/SimpleNonlinearSolve/workflows/Invalidations.yml
new file mode 100644
index 000000000..4d0004e83
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/workflows/Invalidations.yml
@@ -0,0 +1,40 @@
+name: Invalidations
+
+on:
+  pull_request:
+
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: always.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  evaluate:
+    # Only run on PRs to the default branch.
+    # In the PR trigger above branches can be specified only explicitly whereas this check should work for master, main, or any other default branch
+    if: github.base_ref == github.event.repository.default_branch
+    runs-on: ubuntu-latest
+    steps:
+    - uses: julia-actions/setup-julia@v1
+      with:
+        version: '1'
+    - uses: actions/checkout@v3
+    - uses: julia-actions/julia-buildpkg@v1
+    - uses: julia-actions/julia-invalidations@v1
+      id: invs_pr
+
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.repository.default_branch }}
+    - uses: julia-actions/julia-buildpkg@v1
+    - uses: julia-actions/julia-invalidations@v1
+      id: invs_default
+    
+    - name: Report invalidation counts
+      run: |
+        echo "Invalidations on default branch: ${{ steps.invs_default.outputs.total }} (${{ steps.invs_default.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY
+        echo "This branch: ${{ steps.invs_pr.outputs.total }} (${{ steps.invs_pr.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY
+    - name: Check if the PR does increase number of invalidations
+      if: steps.invs_pr.outputs.total > steps.invs_default.outputs.total
+      run: exit 1
diff --git a/lib/SimpleNonlinearSolve/workflows/TagBot.yml b/lib/SimpleNonlinearSolve/workflows/TagBot.yml
new file mode 100644
index 000000000..f49313b66
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/workflows/TagBot.yml
@@ -0,0 +1,15 @@
+name: TagBot
+on:
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
+jobs:
+  TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ssh: ${{ secrets.DOCUMENTER_KEY }}

From 78996f8c4e5320d393466a8c30d890ac645d265b Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 23 Nov 2022 19:17:40 +0100
Subject: [PATCH 003/375] setup with IntervalNonlinearProblem

---
 lib/SimpleNonlinearSolve/Project.toml       |  4 ++--
 lib/SimpleNonlinearSolve/src/ad.jl          | 21 ++++++++++++++++-----
 lib/SimpleNonlinearSolve/src/bisection.jl   |  5 +++--
 lib/SimpleNonlinearSolve/src/falsi.jl       |  9 +++++----
 lib/SimpleNonlinearSolve/src/raphson.jl     |  7 ++++---
 lib/SimpleNonlinearSolve/test/basictests.jl | 14 +++++++-------
 6 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 0e3a98cd2..ca2fe2c05 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,6 +1,6 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
-authors = ["Kanav Gupta <kanav0610@gmail.com>"]
+authors = ["SciML"]
 version = "0.1.0"
 
 [deps]
@@ -21,7 +21,7 @@ FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 RecursiveArrayTools = "2"
 Reexport = "0.2, 1"
-SciMLBase = "1.32"
+SciMLBase = "1.73"
 Setfield = "0.7, 0.8, 1"
 StaticArrays = "0.12,1.0"
 UnPack = "1.0"
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 07a0620d0..501324299 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -1,9 +1,15 @@
 function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     f = prob.f
     p = value(prob.p)
-    u0 = value(prob.u0)
 
-    newprob = NonlinearProblem(f, u0, p; prob.kwargs...)
+    if prob isa IntervalNonlinearProblem
+        tspan = value(prob.tspan)
+        newprob = IntervalNonlinearProblem(f, tspan, p; prob.kwargs...)
+    else
+        u0 = value(prob.u0)
+        newprob = NonlinearProblem(f, u0, p; prob.kwargs...)
+    end
+
     sol = solve(newprob, alg, args...; kwargs...)
 
     uu = sol.u
@@ -39,7 +45,8 @@ end
 
 # avoid ambiguities
 for Alg in [Bisection]
-    @eval function SciMLBase.solve(prob::NonlinearProblem{uType, iip, <:Dual{T, V, P}},
+    @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
+                                                                  <:Dual{T, V, P}},
                                    alg::$Alg, args...;
                                    kwargs...) where {uType, iip, T, V, P}
         sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
@@ -49,8 +56,12 @@ for Alg in [Bisection]
                                         right = Dual{T, V, P}(sol.right, partials))
         #return BracketingSolution(Dual{T,V,P}(sol.left, partials), Dual{T,V,P}(sol.right, partials), sol.retcode, sol.resid)
     end
-    @eval function SciMLBase.solve(prob::NonlinearProblem{uType, iip,
-                                                          <:AbstractArray{<:Dual{T, V, P}}},
+    @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
+                                                                  <:AbstractArray{
+                                                                                  <:Dual{T,
+                                                                                         V,
+                                                                                         P}
+                                                                                  }},
                                    alg::$Alg, args...;
                                    kwargs...) where {uType, iip, T, V, P}
         sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 6e00afba7..2440ea3c0 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -7,10 +7,11 @@ function Bisection(; exact_left = false, exact_right = false)
     Bisection(exact_left, exact_right)
 end
 
-function SciMLBase.solve(prob::NonlinearProblem, alg::Bisection, args...; maxiters = 1000,
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...;
+                         maxiters = 1000,
                          kwargs...)
     f = Base.Fix2(prob.f, prob.p)
-    left, right = prob.u0
+    left, right = prob.tspan
     fl, fr = f(left), f(right)
 
     if iszero(fl)
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index 0536ece19..d431a9251 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -1,9 +1,10 @@
 struct Falsi <: AbstractBracketingAlgorithm end
 
-function SciMLBase.solve(prob::NonlinearProblem, alg::Falsi, args...; maxiters = 1000,
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
+                         maxiters = 1000,
                          kwargs...)
     f = Base.Fix2(prob.f, prob.p)
-    left, right = prob.u0
+    left, right = prob.tspan
     fl, fr = f(left), f(right)
 
     if iszero(fl)
@@ -15,14 +16,14 @@ function SciMLBase.solve(prob::NonlinearProblem, alg::Falsi, args...; maxiters =
     i = 1
     if !iszero(fr)
         while i < maxiters
-            if nextfloat_tdir(left, prob.u0...) == right
+            if nextfloat_tdir(left, prob.tspan...) == right
                 return SciMLBase.build_solution(prob, alg, left, fl;
                                                 retcode = ReturnCode.FloatingPointLimit,
                                                 left = left, right = right)
             end
             mid = (fr * left - fl * right) / (fr - fl)
             for i in 1:10
-                mid = max_tdir(left, prevfloat_tdir(mid, prob.u0...), prob.u0...)
+                mid = max_tdir(left, prevfloat_tdir(mid, prob.tspan...), prob.tspan...)
             end
             if mid == right || mid == left
                 break
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 8ed325a86..6a5d235e0 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -1,13 +1,14 @@
 struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     function SimpleNewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
-                                   diff_type = Val{:forward})
+                                 diff_type = Val{:forward})
         new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
-        SciMLBase._unwrap_val(diff_type)}()
+            SciMLBase._unwrap_val(diff_type)}()
     end
 end
 
 function SciMLBase.solve(prob::NonlinearProblem,
-                         alg::SimpleNewtonRaphson, args...; xatol = nothing, xrtol = nothing,
+                         alg::SimpleNewtonRaphson, args...; xatol = nothing,
+                         xrtol = nothing,
                          maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 47143dbce..0a0716173 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -57,10 +57,10 @@ for p in 1.1:0.1:100.0
     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
 end
 
-u0 = (1.0, 20.0)
+tspan = (1.0, 20.0)
 # Falsi
 g = function (p)
-    probN = NonlinearProblem{false}(f, typeof(p).(u0), p)
+    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
     sol = solve(probN, Falsi())
     return sol.left
 end
@@ -70,13 +70,13 @@ for p in 1.1:0.1:100.0
     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
 end
 
-f, u0 = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
+f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
 t = (p) -> [sqrt(p[2] / p[1])]
 p = [0.9, 50.0]
 for alg in [Bisection(), Falsi()]
     global g, p
     g = function (p)
-        probN = NonlinearProblem{false}(f, u0, p)
+        probN = IntervalNonlinearProblem{false}(f, tspan, p)
         sol = solve(probN, Bisection())
         return [sol.left]
     end
@@ -115,8 +115,8 @@ for u0 in [1.0, [1, 1.0]]
 end
 
 # Bisection Tests
-f, u0 = (u, p) -> u .* u .- 2.0, (1.0, 2.0)
-probB = NonlinearProblem(f, u0)
+f, tspan = (u, p) -> u .* u .- 2.0, (1.0, 2.0)
+probB = IntervalNonlinearProblem(f, tspan)
 
 # Falsi
 sol = solve(probB, Falsi())
@@ -135,7 +135,7 @@ f = function (u, p)
         return 0.0
     end
 end
-probB = NonlinearProblem(f, (0.0, 4.0))
+probB = IntervalNonlinearProblem(f, (0.0, 4.0))
 
 sol = solve(probB, Bisection(; exact_left = true))
 @test f(sol.left, nothing) < 0.0

From 7c6e65419c621940b67cf913277c80c202f8cb88 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 23 Nov 2022 19:19:04 +0100
Subject: [PATCH 004/375] fix workflow location

---
 lib/SimpleNonlinearSolve/{ => .github}/workflows/CI.yml           | 0
 lib/SimpleNonlinearSolve/{ => .github}/workflows/CompatHelper.yml | 0
 .../{ => .github}/workflows/Documentation.yml                     | 0
 lib/SimpleNonlinearSolve/{ => .github}/workflows/Downstream.yml   | 0
 lib/SimpleNonlinearSolve/{ => .github}/workflows/FormatCheck.yml  | 0
 .../{ => .github}/workflows/Invalidations.yml                     | 0
 lib/SimpleNonlinearSolve/{ => .github}/workflows/TagBot.yml       | 0
 7 files changed, 0 insertions(+), 0 deletions(-)
 rename lib/SimpleNonlinearSolve/{ => .github}/workflows/CI.yml (100%)
 rename lib/SimpleNonlinearSolve/{ => .github}/workflows/CompatHelper.yml (100%)
 rename lib/SimpleNonlinearSolve/{ => .github}/workflows/Documentation.yml (100%)
 rename lib/SimpleNonlinearSolve/{ => .github}/workflows/Downstream.yml (100%)
 rename lib/SimpleNonlinearSolve/{ => .github}/workflows/FormatCheck.yml (100%)
 rename lib/SimpleNonlinearSolve/{ => .github}/workflows/Invalidations.yml (100%)
 rename lib/SimpleNonlinearSolve/{ => .github}/workflows/TagBot.yml (100%)

diff --git a/lib/SimpleNonlinearSolve/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
similarity index 100%
rename from lib/SimpleNonlinearSolve/workflows/CI.yml
rename to lib/SimpleNonlinearSolve/.github/workflows/CI.yml
diff --git a/lib/SimpleNonlinearSolve/workflows/CompatHelper.yml b/lib/SimpleNonlinearSolve/.github/workflows/CompatHelper.yml
similarity index 100%
rename from lib/SimpleNonlinearSolve/workflows/CompatHelper.yml
rename to lib/SimpleNonlinearSolve/.github/workflows/CompatHelper.yml
diff --git a/lib/SimpleNonlinearSolve/workflows/Documentation.yml b/lib/SimpleNonlinearSolve/.github/workflows/Documentation.yml
similarity index 100%
rename from lib/SimpleNonlinearSolve/workflows/Documentation.yml
rename to lib/SimpleNonlinearSolve/.github/workflows/Documentation.yml
diff --git a/lib/SimpleNonlinearSolve/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
similarity index 100%
rename from lib/SimpleNonlinearSolve/workflows/Downstream.yml
rename to lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
diff --git a/lib/SimpleNonlinearSolve/workflows/FormatCheck.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
similarity index 100%
rename from lib/SimpleNonlinearSolve/workflows/FormatCheck.yml
rename to lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
diff --git a/lib/SimpleNonlinearSolve/workflows/Invalidations.yml b/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
similarity index 100%
rename from lib/SimpleNonlinearSolve/workflows/Invalidations.yml
rename to lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
diff --git a/lib/SimpleNonlinearSolve/workflows/TagBot.yml b/lib/SimpleNonlinearSolve/.github/workflows/TagBot.yml
similarity index 100%
rename from lib/SimpleNonlinearSolve/workflows/TagBot.yml
rename to lib/SimpleNonlinearSolve/.github/workflows/TagBot.yml

From 8a56da945e0ab04cd02610af3c6d2be1406e73dd Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 23 Nov 2022 19:39:15 +0100
Subject: [PATCH 005/375] fix tolerances

---
 lib/SimpleNonlinearSolve/src/raphson.jl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 6a5d235e0..1f23debd7 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -7,8 +7,8 @@ struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
 end
 
 function SciMLBase.solve(prob::NonlinearProblem,
-                         alg::SimpleNewtonRaphson, args...; xatol = nothing,
-                         xrtol = nothing,
+                         alg::SimpleNewtonRaphson, args...; abstol = nothing,
+                         reltol = nothing,
                          maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
@@ -19,8 +19,8 @@ function SciMLBase.solve(prob::NonlinearProblem,
         error("SimpleNewtonRaphson currently only supports out-of-place nonlinear problems")
     end
 
-    atol = xatol !== nothing ? xatol : oneunit(eltype(T)) * (eps(one(eltype(T))))^(4 // 5)
-    rtol = xrtol !== nothing ? xrtol : eps(one(eltype(T)))^(4 // 5)
+    atol = abstol !== nothing ? abstol : oneunit(eltype(T)) * (eps(one(eltype(T))))^(4 // 5)
+    rtol = reltol !== nothing ? reltol : eps(one(eltype(T)))^(4 // 5)
 
     if typeof(x) <: Number
         xo = oftype(one(eltype(x)), Inf)

From b7b67719f1794120b144dd0e87e5fc276e5887f8 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 23 Nov 2022 19:45:54 +0100
Subject: [PATCH 006/375] master -> main

---
 .../.github/workflows/CI.yml                  |  4 +--
 .../.github/workflows/Documentation.yml       | 29 -------------------
 .../.github/workflows/Downstream.yml          |  2 +-
 .../.github/workflows/FormatCheck.yml         |  2 +-
 4 files changed, 4 insertions(+), 33 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/.github/workflows/Documentation.yml

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index c8d041ecf..80a2aea7b 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -2,10 +2,10 @@ name: CI
 on:
   pull_request:
     branches:
-      - master
+      - main
   push:
     branches:
-      - master
+      - main
 jobs:
   test:
     runs-on: ubuntu-latest
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Documentation.yml b/lib/SimpleNonlinearSolve/.github/workflows/Documentation.yml
deleted file mode 100644
index f64a315b6..000000000
--- a/lib/SimpleNonlinearSolve/.github/workflows/Documentation.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-name: Documentation
-
-on:
-  push:
-    branches:
-      - master
-      - 'release-'
-    tags: '*'
-  pull_request:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@latest
-        with:
-          version: '1'
-      - name: Install dependencies
-        run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
-      - name: Build and deploy
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For authentication with GitHub Actions token
-          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key
-        run: julia --project=docs/ --code-coverage=user docs/make.jl
-      - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v1
-        with:
-          file: lcov.info
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
index 916a33b4b..122cccf4d 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
@@ -1,7 +1,7 @@
 name: IntegrationTest
 on:
   push:
-    branches: [master]
+    branches: [main]
     tags: [v*]
   pull_request:
 
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
index 2a3517a0f..e4e3512e2 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
@@ -3,7 +3,7 @@ name: format-check
 on:
   push:
     branches:
-      - 'master'
+      - 'main'
       - 'release-'
     tags: '*'
   pull_request:

From 4dcd7a76853dc8ac94e73374c3caa284a548dce0 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 23 Nov 2022 22:37:17 +0100
Subject: [PATCH 007/375] better readme

---
 lib/SimpleNonlinearSolve/README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index ef305c0c8..a7febbeb4 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -9,10 +9,10 @@
 [![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac)
 [![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle)
 
-
-
-
 Fast implementations of root finding algorithms in Julia that satisfy the SciML common interface.
+SimpleNonlinearSolve.jl focuses on low-dependency implementations of very fast methods for
+very small and simple problems. For the full set of solvers, see NonlinearSolve.jl, of which
+SimpleNonlinearSolve.jl is just one solver set.
 
 For information on using the package,
 [see the stable documentation](https://docs.sciml.ai/NonlinearSolve/stable/). Use the
@@ -22,17 +22,17 @@ the documentation which contains the unreleased features.
 ## High Level Examples
 
 ```julia
-using NonlinearSolve, StaticArrays
+using SimpleNonlinearSolve, StaticArrays
 
 f(u,p) = u .* u .- 2
 u0 = @SVector[1.0, 1.0]
 probN = NonlinearProblem{false}(f, u0)
-solver = solve(probN, NewtonRaphson(), tol = 1e-9)
+solver = solve(probN, SimpleNewtonRaphson(), tol = 1e-9)
 
 ## Bracketing Methods
 
 f(u, p) = u .* u .- 2.0
 u0 = (1.0, 2.0) # brackets
-probB = NonlinearProblem(f, u0)
+probB = IntervalNonlinearProblem(f, u0)
 sol = solve(probB, Falsi())
 ```

From 26cee49125e1ac6cabce0962bfebd02255184bda Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 24 Nov 2022 10:14:37 +0100
Subject: [PATCH 008/375] Dramatically reduce dependencies

---
 lib/SimpleNonlinearSolve/Project.toml           | 17 +++++------------
 .../src/SimpleNonlinearSolve.jl                 |  6 +-----
 lib/SimpleNonlinearSolve/src/ad.jl              |  6 ++++--
 3 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index ca2fe2c05..0c9ebc040 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,38 +1,31 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.0"
+version = "0.1.1"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
-LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
-Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
+StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [compat]
 ArrayInterfaceCore = "0.1.1"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
-RecursiveArrayTools = "2"
 Reexport = "0.2, 1"
 SciMLBase = "1.73"
-Setfield = "0.7, 0.8, 1"
-StaticArrays = "0.12,1.0"
-UnPack = "1.0"
+StaticArraysCore = "1.4"
 julia = "1.6"
 
 [extras]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "ForwardDiff"]
+test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays"]
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 191bb8731..164f99cf8 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -1,13 +1,9 @@
 module SimpleNonlinearSolve
 
 using Reexport
-using UnPack: @unpack
 using FiniteDiff, ForwardDiff
 using ForwardDiff: Dual
-using Setfield
-using StaticArrays
-using RecursiveArrayTools
-using LinearAlgebra
+using StaticArraysCore
 import ArrayInterfaceCore
 
 @reexport using SciMLBase
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 501324299..3b8d3fee0 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -28,14 +28,16 @@ function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return sol, partials
 end
 
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, SVector}, iip,
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
+                                                iip,
                                                 <:Dual{T, V, P}}, alg::SimpleNewtonRaphson,
                          args...; kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
                                     retcode = sol.retcode)
 end
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, SVector}, iip,
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
+                                                iip,
                                                 <:AbstractArray{<:Dual{T, V, P}}},
                          alg::SimpleNewtonRaphson, args...; kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)

From 41a1d01ce3ede0d1b0fe13a8e6f0b69b4a2bc855 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 24 Nov 2022 10:37:29 +0100
Subject: [PATCH 009/375] snoopprecompile

---
 lib/SimpleNonlinearSolve/Project.toml         |  2 ++
 .../src/SimpleNonlinearSolve.jl               | 24 +++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 0c9ebc040..71f516462 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -9,6 +9,7 @@ FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [compat]
@@ -17,6 +18,7 @@ FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 Reexport = "0.2, 1"
 SciMLBase = "1.73"
+SnoopPrecompile = "1"
 StaticArraysCore = "1.4"
 julia = "1.6"
 
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 164f99cf8..bd4788193 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -19,6 +19,30 @@ include("falsi.jl")
 include("raphson.jl")
 include("ad.jl")
 
+import SnoopPrecompile
+
+SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
+    prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
+    for alg in (SimpleNewtonRaphson,)
+        solve(prob_no_brack, alg(), tol = T(1e-2))
+    end
+
+    #=
+    for alg in (SimpleNewtonRaphson,)
+        for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
+            u0 = T.(.1)
+            probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
+            solve(probN, alg(), tol = T(1e-2))
+        end
+    end
+    =#
+
+    prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p, T.((0.0, 2.0)), T(2))
+    for alg in (Bisection, Falsi)
+        solve(prob_brack, alg(), tol = T(1e-2))
+    end
+end end
+
 # DiffEq styled algorithms
 export Bisection, Falsi, SimpleNewtonRaphson
 

From 9667029500fff3a92b97767ba9f6ecdb35dc7e7c Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 24 Nov 2022 10:55:52 +0100
Subject: [PATCH 010/375] Fix return codes

---
 lib/SimpleNonlinearSolve/Project.toml       | 2 +-
 lib/SimpleNonlinearSolve/src/raphson.jl     | 4 ++--
 lib/SimpleNonlinearSolve/test/basictests.jl | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 71f516462..8b87181da 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.1"
+version = "0.1.2"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 1f23debd7..3db5b62bc 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -40,11 +40,11 @@ function SciMLBase.solve(prob::NonlinearProblem,
                                                           fx)
         end
         iszero(fx) &&
-            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Default)
+            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
         Δx = dfx \ fx
         x -= Δx
         if isapprox(x, xo, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Default)
+            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
         end
         xo = x
     end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 0a0716173..0c4194675 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -18,7 +18,7 @@ end
 const csu0 = 1.0
 
 sol = benchmark_scalar(sf, csu0)
-@test sol.retcode === ReturnCode.Default
+@test sol.retcode === ReturnCode.Success
 @test sol.u * sol.u - 2 < 1e-9
 
 @test (@ballocated benchmark_scalar(sf, csu0)) == 0

From 70edef753fdf52eda16db1bd1b0d98e71f718460 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 24 Nov 2022 11:07:36 +0100
Subject: [PATCH 011/375] Update Downstream.yml

---
 lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
index 122cccf4d..6abfcfddc 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
@@ -18,6 +18,12 @@ jobs:
         os: [ubuntu-latest]
         package:
           - {user: SciML, repo: ModelingToolkit.jl, group: All}
+          - {user: SciML, repo: DiffEqBase.jl, group: Core}
+          - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceI}
+          - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceII}
+          - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceIII}
+          - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceIV}
+          - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceV}
     steps:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1

From 5134d41672be3a274712bb8d51787311c7a3ce64 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 24 Nov 2022 15:31:27 +0100
Subject: [PATCH 012/375] Update README.md

---
 lib/SimpleNonlinearSolve/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index a7febbeb4..818872c0b 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -1,4 +1,4 @@
-# NonlinearSolve.jl
+# SimpleNonlinearSolve.jl
 
 [![Join the chat at https://julialang.zulipchat.com #sciml-bridged](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/279055-sciml-bridged)
 [![Global Docs](https://img.shields.io/badge/docs-SciML-blue.svg)](https://docs.sciml.ai/NonlinearSolve/stable/)
@@ -11,7 +11,8 @@
 
 Fast implementations of root finding algorithms in Julia that satisfy the SciML common interface.
 SimpleNonlinearSolve.jl focuses on low-dependency implementations of very fast methods for
-very small and simple problems. For the full set of solvers, see NonlinearSolve.jl, of which
+very small and simple problems. For the full set of solvers, see 
+[NonlinearSolve.jl](https://github.com/SciML/NonlinearSolve.jl), of which
 SimpleNonlinearSolve.jl is just one solver set.
 
 For information on using the package,

From 0bd99235ed3655dde1cc52eabcd89cff721e73cf Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 24 Nov 2022 21:04:13 +0100
Subject: [PATCH 013/375] Update CompatHelper.yml

---
 lib/SimpleNonlinearSolve/.github/workflows/CompatHelper.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CompatHelper.yml b/lib/SimpleNonlinearSolve/.github/workflows/CompatHelper.yml
index 73494545f..0fe6c3748 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CompatHelper.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CompatHelper.yml
@@ -23,4 +23,4 @@ jobs:
       - name: CompatHelper.main()
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: julia -e 'using CompatHelper; CompatHelper.main(;subdirs=["", "docs"])'
+        run: julia -e 'using CompatHelper; CompatHelper.main()'

From 92732e73d7c28c6bd41bd11da00e1bd540728cf1 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Sun, 4 Dec 2022 13:46:21 +0100
Subject: [PATCH 014/375] improve docstrings

---
 lib/SimpleNonlinearSolve/Project.toml     |  2 +-
 lib/SimpleNonlinearSolve/src/bisection.jl | 12 +++++++++
 lib/SimpleNonlinearSolve/src/falsi.jl     |  3 +++
 lib/SimpleNonlinearSolve/src/raphson.jl   | 30 +++++++++++++++++++++++
 4 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 8b87181da..f04d3a6c3 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.2"
+version = "0.1.3"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 2440ea3c0..e90f28f9e 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -1,3 +1,15 @@
+"""
+`Bisection(; exact_left = false, exact_right = false)`
+
+A common bisection method.
+
+### Keyword Arguments
+
+- `exact_left`: whether to enforce whether the left side of the interval must be exactly
+  zero for the returned result. Defualts to false.
+- `exact_right`: whether to enforce whether the right side of the interval must be exactly
+  zero for the returned result. Defualts to false.
+"""
 struct Bisection <: AbstractBracketingAlgorithm
     exact_left::Bool
     exact_right::Bool
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index d431a9251..81ce68ffa 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -1,3 +1,6 @@
+"""
+`Falsi`: A non-allocating regula falsi method
+"""
 struct Falsi <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 3db5b62bc..ed00fad30 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -1,3 +1,33 @@
+"""
+```julia
+SimpleNewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
+                                 diff_type = Val{:forward})
+```
+
+A low-overhead implementation of Newton-Raphson. This method is non-allocating on scalar
+and static array problems.
+
+!!! note
+
+    As part of the decreased overhead, this method omits some of the higher level error
+    catching of the other methods. Thus to see better error messages, use one of the other
+    methods like `NewtonRaphson`
+
+### Keyword Arguments
+
+- `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
+  system. This allows for multiple derivative columns to be computed simultaniously,
+  improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
+  default chunk size mechanism. For more details, see the documentation for
+  [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
+- `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
+  Note that this argument is ignored if an analytical Jacobian is passed as that will be
+  used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
+  If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
+- `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
+  `Val{:forward}` for forward finite differences. For more details on the choices, see the
+  [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+"""
 struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     function SimpleNewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
                                  diff_type = Val{:forward})

From db4cf617354e5f825db181ce3cf39f9f83cd357f Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 24 Dec 2022 10:24:57 -0500
Subject: [PATCH 015/375] Support complex in newton

---
 lib/SimpleNonlinearSolve/Project.toml   | 2 +-
 lib/SimpleNonlinearSolve/src/raphson.jl | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index f04d3a6c3..fa4f7f015 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.3"
+version = "0.1.4"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index ed00fad30..2848d6d1a 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -49,8 +49,8 @@ function SciMLBase.solve(prob::NonlinearProblem,
         error("SimpleNewtonRaphson currently only supports out-of-place nonlinear problems")
     end
 
-    atol = abstol !== nothing ? abstol : oneunit(eltype(T)) * (eps(one(eltype(T))))^(4 // 5)
-    rtol = reltol !== nothing ? reltol : eps(one(eltype(T)))^(4 // 5)
+    atol = abstol !== nothing ? abstol : real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
+    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
 
     if typeof(x) <: Number
         xo = oftype(one(eltype(x)), Inf)

From 28ac6845c3cac8c246dc47c7df0982e200c3a5e9 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 24 Dec 2022 10:42:51 -0500
Subject: [PATCH 016/375] format

---
 lib/SimpleNonlinearSolve/src/raphson.jl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 2848d6d1a..47ae6eaf8 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -49,7 +49,8 @@ function SciMLBase.solve(prob::NonlinearProblem,
         error("SimpleNewtonRaphson currently only supports out-of-place nonlinear problems")
     end
 
-    atol = abstol !== nothing ? abstol : real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
+    atol = abstol !== nothing ? abstol :
+           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
     rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
 
     if typeof(x) <: Number
@@ -78,5 +79,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
         end
         xo = x
     end
+
+    @show x, fx
     return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end

From 15c2593e652e03a1821730b078a1ac2df78326cd Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Sat, 31 Dec 2022 06:24:29 +0100
Subject: [PATCH 017/375] inital commit, started to implement Broyden

---
 .../src/SimpleNonlinearSolve.jl               |  6 ++-
 lib/SimpleNonlinearSolve/src/broyden.jl       | 43 +++++++++++++++++++
 lib/SimpleNonlinearSolve/test/basictests.jl   |  4 +-
 3 files changed, 50 insertions(+), 3 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/broyden.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index bd4788193..213ba2317 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -4,6 +4,7 @@ using Reexport
 using FiniteDiff, ForwardDiff
 using ForwardDiff: Dual
 using StaticArraysCore
+using LinearAlgebra # TODO check if it is ok to add this
 import ArrayInterfaceCore
 
 @reexport using SciMLBase
@@ -18,12 +19,13 @@ include("bisection.jl")
 include("falsi.jl")
 include("raphson.jl")
 include("ad.jl")
+include("broyden.jl")
 
 import SnoopPrecompile
 
 SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-    for alg in (SimpleNewtonRaphson,)
+    for alg in (SimpleNewtonRaphson, Broyden)
         solve(prob_no_brack, alg(), tol = T(1e-2))
     end
 
@@ -44,6 +46,6 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
 end end
 
 # DiffEq styled algorithms
-export Bisection, Falsi, SimpleNewtonRaphson
+export Bisection, Broyden, Falsi, SimpleNewtonRaphson
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
new file mode 100644
index 000000000..9f9388bad
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -0,0 +1,43 @@
+# TODO add docstrings
+
+# TODO check what the supertype should be
+# TODO check if this should be defined as in raphson.jl
+struct Broyden <: AbstractSimpleNonlinearSolveAlgorithm end
+
+function SciMLBase.solve(prob::NonlinearProblem,
+                         alg::Broyden, args...; abstol = nothing,
+                         reltol = nothing,
+                         maxiters = 1000, kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    xₙ = float(prob.u0)
+    T = typeof(xₙ)
+    J⁻¹ = Matrix{T}(I, length(xₙ), length(xₙ))
+
+    if SciMLBase.isinplace(prob)
+        error("Broyden currently only supports out-of-place nonlinear problems")
+    end
+
+    atol = abstol !== nothing ? abstol :
+           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
+    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+
+    for _ in 1:maxiters
+        # TODO check if nameing with heavy use of subscrips is ok
+        fₙ = f(xₙ)
+        xₙ₊₁ = xₙ + J⁻¹ * fₙ
+        Δxₙ = xₙ₊₁ - xₙ
+        Δfₙ = f(xₙ₊₁) - fₙ
+        J⁻¹ .+= ((Δxₙ .- J⁻¹ * Δfₙ) ./ (Δxₙ' * J⁻¹ * Δfₙ)) * (Δxₙ' * J⁻¹)
+
+        iszero(fₙ) &&
+            return SciMLBase.build_solution(prob, alg, xₙ₊₁, fₙ; retcode = ReturnCode.Success)
+
+        if isapprox(xₙ₊₁, xₙ, atol = atol, rtol = rtol)
+            return SciMLBase.build_solution(prob, alg, xₙ₊₁, fₙ; retcode = ReturnCode.Success)
+        end
+        xₙ = xₙ₊₁
+    end
+
+    @show xₙ, fₙ
+    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 0c4194675..bc5a3902a 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -77,7 +77,7 @@ for alg in [Bisection(), Falsi()]
     global g, p
     g = function (p)
         probN = IntervalNonlinearProblem{false}(f, tspan, p)
-        sol = solve(probN, Bisection())
+        sol = solve(probN, Bisection())  # TODO check if "alg" should replace "Bisection()"
         return [sol.left]
     end
 
@@ -102,6 +102,7 @@ probN = NonlinearProblem(f, u0)
 @test solve(probN, SimpleNewtonRaphson(); immutable = false).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
+# TODO check why the 2 lines above are identical
 
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol
@@ -109,6 +110,7 @@ for u0 in [1.0, [1, 1.0]]
     probN = NonlinearProblem(f, u0)
     sol = sqrt(2) * u0
 
+    # TODO check why the two lines below are identical
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol

From 2d0c7d55a428019d78f259a3a890d2bb8ea67d83 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Mon, 2 Jan 2023 20:20:26 +0100
Subject: [PATCH 018/375] Implemented Broyden and tests

---
 lib/SimpleNonlinearSolve/Project.toml       |  1 +
 lib/SimpleNonlinearSolve/src/broyden.jl     | 45 ++++++++-----
 lib/SimpleNonlinearSolve/test/basictests.jl | 73 +++++++++++++--------
 3 files changed, 77 insertions(+), 42 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index fa4f7f015..3f4c0d749 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -7,6 +7,7 @@ version = "0.1.4"
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 9f9388bad..7664cf6c8 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -5,13 +5,19 @@
 struct Broyden <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.solve(prob::NonlinearProblem,
-                         alg::Broyden, args...; abstol = nothing,
-                         reltol = nothing,
-                         maxiters = 1000, kwargs...)
+    alg::Broyden, args...; abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000, kwargs...)
+
     f = Base.Fix2(prob.f, prob.p)
-    xₙ = float(prob.u0)
-    T = typeof(xₙ)
-    J⁻¹ = Matrix{T}(I, length(xₙ), length(xₙ))
+    x = float(prob.u0)
+    fₙ = f(x)
+    T = eltype(x)
+    if length(x) > 1
+        J⁻¹ = Matrix{T}(I, length(x), length(x))
+    else
+        J⁻¹ = 1
+    end
 
     if SciMLBase.isinplace(prob)
         error("Broyden currently only supports out-of-place nonlinear problems")
@@ -21,23 +27,30 @@ function SciMLBase.solve(prob::NonlinearProblem,
            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
     rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
 
-    for _ in 1:maxiters
-        # TODO check if nameing with heavy use of subscrips is ok
+    xₙ = x
+    xₙ₋₁ = x
+    fₙ₋₁ = fₙ
+    for n in 1:maxiters
+
+        xₙ = xₙ₋₁ - J⁻¹ * fₙ₋₁
         fₙ = f(xₙ)
-        xₙ₊₁ = xₙ + J⁻¹ * fₙ
-        Δxₙ = xₙ₊₁ - xₙ
-        Δfₙ = f(xₙ₊₁) - fₙ
-        J⁻¹ .+= ((Δxₙ .- J⁻¹ * Δfₙ) ./ (Δxₙ' * J⁻¹ * Δfₙ)) * (Δxₙ' * J⁻¹)
+        Δxₙ = xₙ - xₙ₋₁
+        Δfₙ = fₙ - fₙ₋₁
+        J⁻¹ += ((Δxₙ - J⁻¹ * Δfₙ) ./ (Δxₙ' * J⁻¹ * Δfₙ)) * (Δxₙ' * J⁻¹)
 
         iszero(fₙ) &&
-            return SciMLBase.build_solution(prob, alg, xₙ₊₁, fₙ; retcode = ReturnCode.Success)
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                            retcode = ReturnCode.Success)
 
-        if isapprox(xₙ₊₁, xₙ, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, xₙ₊₁, fₙ; retcode = ReturnCode.Success)
+        if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                            retcode = ReturnCode.Success)
         end
-        xₙ = xₙ₊₁
+        xₙ₋₁ = xₙ
+        fₙ₋₁ = fₙ
     end
 
     @show xₙ, fₙ
+
     return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index bc5a3902a..5518fb47c 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -3,6 +3,7 @@ using StaticArrays
 using BenchmarkTools
 using Test
 
+# SimpleNewtonRaphson
 function benchmark_scalar(f, u0)
     probN = NonlinearProblem{false}(f, u0)
     sol = (solve(probN, SimpleNewtonRaphson()))
@@ -23,38 +24,53 @@ sol = benchmark_scalar(sf, csu0)
 
 @test (@ballocated benchmark_scalar(sf, csu0)) == 0
 
+# Broyden
+function benchmark_scalar(f, u0)
+    probN = NonlinearProblem{false}(f, u0)
+    sol = (solve(probN, Broyden()))
+end
+
+sol = benchmark_scalar(sf, csu0)
+@test sol.retcode === ReturnCode.Success
+@test sol.u * sol.u - 2 < 1e-9
+@test (@ballocated benchmark_scalar(sf, csu0)) == 0
+
 # AD Tests
 using ForwardDiff
 
 # Immutable
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
-g = function (p)
-    probN = NonlinearProblem{false}(f, csu0, p)
-    sol = solve(probN, SimpleNewtonRaphson(), tol = 1e-9)
-    return sol.u[end]
-end
+for alg in [SimpleNewtonRaphson(), Broyden()]
+    g = function (p)
+        probN = NonlinearProblem{false}(f, csu0, p)
+        sol = solve(probN, alg, tol = 1e-9)
+        return sol.u[end]
+    end
 
-for p in 1.0:0.1:100.0
-    @test g(p) ≈ sqrt(p)
-    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+    for p in 1.1:0.1:100.0
+        @test g(p) ≈ sqrt(p)
+        @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+    end
 end
 
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
+for alg in [SimpleNewtonRaphson(), Broyden()]
 
-# SimpleNewtonRaphson
-g = function (p)
-    probN = NonlinearProblem{false}(f, oftype(p, u0), p)
-    sol = solve(probN, SimpleNewtonRaphson())
-    return sol.u
-end
-
-@test ForwardDiff.derivative(g, 1.0) ≈ 0.5
+    g = function (p)
+        probN = NonlinearProblem{false}(f, oftype(p, u0), p)
+        sol = solve(probN, alg)
+        return sol.u
+    end
 
-for p in 1.1:0.1:100.0
-    @test g(p) ≈ sqrt(p)
+    p = 1.1
     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+
+    for p in 1.1:0.1:100.0
+        @test g(p) ≈ sqrt(p)
+        @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+    end
 end
 
 tspan = (1.0, 20.0)
@@ -77,7 +93,7 @@ for alg in [Bisection(), Falsi()]
     global g, p
     g = function (p)
         probN = IntervalNonlinearProblem{false}(f, tspan, p)
-        sol = solve(probN, Bisection())  # TODO check if "alg" should replace "Bisection()"
+        sol = solve(probN, alg)
         return [sol.left]
     end
 
@@ -85,16 +101,18 @@ for alg in [Bisection(), Falsi()]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
 
-gnewton = function (p)
-    probN = NonlinearProblem{false}(f, 0.5, p)
-    sol = solve(probN, SimpleNewtonRaphson())
-    return [sol.u]
+for alg in [SimpleNewtonRaphson(), Broyden()]
+    global g, p
+    g = function (p)
+        probN = NonlinearProblem{false}(f, 0.5, p)
+        sol = solve(probN, alg)
+        return [sol.u]
+    end
+    @test g(p) ≈ [sqrt(p[2] / p[1])]
+    @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
-@test gnewton(p) ≈ [sqrt(p[2] / p[1])]
-@test ForwardDiff.jacobian(gnewton, p) ≈ ForwardDiff.jacobian(t, p)
 
 # Error Checks
-
 f, u0 = (u, p) -> u .* u .- 2.0, @SVector[1.0, 1.0]
 probN = NonlinearProblem(f, u0)
 
@@ -103,6 +121,8 @@ probN = NonlinearProblem(f, u0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
 # TODO check why the 2 lines above are identical
+@test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
+@test solve(probN, Broyden(); immutable = false).u[end] ≈ sqrt(2.0)
 
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol
@@ -114,6 +134,7 @@ for u0 in [1.0, [1, 1.0]]
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol
+    @test solve(probN, Broyden()).u ≈ sol
 end
 
 # Bisection Tests

From 37522c934da12068ac02d5b401cc0f18449ba043 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Mon, 2 Jan 2023 22:24:29 +0100
Subject: [PATCH 019/375] Cleanup

---
 .../src/SimpleNonlinearSolve.jl               |  2 +-
 lib/SimpleNonlinearSolve/src/broyden.jl       | 21 +++++++++++--------
 lib/SimpleNonlinearSolve/test/basictests.jl   |  8 ++-----
 3 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 213ba2317..0b2fd0b48 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -4,7 +4,7 @@ using Reexport
 using FiniteDiff, ForwardDiff
 using ForwardDiff: Dual
 using StaticArraysCore
-using LinearAlgebra # TODO check if it is ok to add this
+using LinearAlgebra
 import ArrayInterfaceCore
 
 @reexport using SciMLBase
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 7664cf6c8..35d4adbbc 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -1,14 +1,18 @@
-# TODO add docstrings
+"""
+```julia
+Broyden()
+```
+
+A low-overhead implementation of Broyden. This method is non-allocating on scalar
+and static array problems.
+"""
 
-# TODO check what the supertype should be
-# TODO check if this should be defined as in raphson.jl
 struct Broyden <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.solve(prob::NonlinearProblem,
-    alg::Broyden, args...; abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000, kwargs...)
-
+                         alg::Broyden, args...; abstol = nothing,
+                         reltol = nothing,
+                         maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fₙ = f(x)
@@ -30,8 +34,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     xₙ = x
     xₙ₋₁ = x
     fₙ₋₁ = fₙ
-    for n in 1:maxiters
-
+    for _ in 1:maxiters
         xₙ = xₙ₋₁ - J⁻¹ * fₙ₋₁
         fₙ = f(xₙ)
         Δxₙ = xₙ - xₙ₋₁
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 5518fb47c..6a5f69b3f 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -57,16 +57,12 @@ end
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
 for alg in [SimpleNewtonRaphson(), Broyden()]
-
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
         return sol.u
     end
 
-    p = 1.1
-    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-
     for p in 1.1:0.1:100.0
         @test g(p) ≈ sqrt(p)
         @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
@@ -120,7 +116,7 @@ probN = NonlinearProblem(f, u0)
 @test solve(probN, SimpleNewtonRaphson(); immutable = false).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
-# TODO check why the 2 lines above are identical
+
 @test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
 @test solve(probN, Broyden(); immutable = false).u[end] ≈ sqrt(2.0)
 
@@ -130,10 +126,10 @@ for u0 in [1.0, [1, 1.0]]
     probN = NonlinearProblem(f, u0)
     sol = sqrt(2) * u0
 
-    # TODO check why the two lines below are identical
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol
+
     @test solve(probN, Broyden()).u ≈ sol
 end
 

From 93750f5c78e6dc26f013363e9a13779ef4c03650 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 2 Jan 2023 17:28:03 -0500
Subject: [PATCH 020/375] Update src/broyden.jl

---
 lib/SimpleNonlinearSolve/src/broyden.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 35d4adbbc..242ec674e 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -6,7 +6,6 @@ Broyden()
 A low-overhead implementation of Broyden. This method is non-allocating on scalar
 and static array problems.
 """
-
 struct Broyden <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.solve(prob::NonlinearProblem,

From 6c673bf4a0b3eabcee1583f9b644ce941a739a35 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 2 Jan 2023 17:34:06 -0500
Subject: [PATCH 021/375] Update src/broyden.jl

---
 lib/SimpleNonlinearSolve/src/broyden.jl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 242ec674e..ddeba8d1d 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -52,7 +52,5 @@ function SciMLBase.solve(prob::NonlinearProblem,
         fₙ₋₁ = fₙ
     end
 
-    @show xₙ, fₙ
-
     return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
 end

From f0d69b08abd916ae6e15db76e32699298614bdce Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 2 Jan 2023 17:43:43 -0500
Subject: [PATCH 022/375] Update src/broyden.jl

---
 lib/SimpleNonlinearSolve/src/broyden.jl | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index ddeba8d1d..ab4d313b9 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -16,11 +16,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     x = float(prob.u0)
     fₙ = f(x)
     T = eltype(x)
-    if length(x) > 1
-        J⁻¹ = Matrix{T}(I, length(x), length(x))
-    else
-        J⁻¹ = 1
-    end
+    J⁻¹ = ArrayInterfaceCore.zeromatrix(x)
 
     if SciMLBase.isinplace(prob)
         error("Broyden currently only supports out-of-place nonlinear problems")

From ae9d41774c4cdedd88a78d4be121d7170e087948 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 2 Jan 2023 17:44:26 -0500
Subject: [PATCH 023/375] Update src/broyden.jl

---
 lib/SimpleNonlinearSolve/src/broyden.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index ab4d313b9..49f53950a 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -16,7 +16,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     x = float(prob.u0)
     fₙ = f(x)
     T = eltype(x)
-    J⁻¹ = ArrayInterfaceCore.zeromatrix(x)
+    J⁻¹ = ArrayInterfaceCore.zeromatrix(x) + I
 
     if SciMLBase.isinplace(prob)
         error("Broyden currently only supports out-of-place nonlinear problems")

From 8aa03478958812b50065c36b8ae564e0d03b339c Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Tue, 3 Jan 2023 17:33:45 +0100
Subject: [PATCH 024/375] Implemented and tested the Klement solver

---
 .../src/SimpleNonlinearSolve.jl               |  3 +-
 lib/SimpleNonlinearSolve/src/klement.jl       | 65 +++++++++++++++++++
 lib/SimpleNonlinearSolve/test/basictests.jl   | 22 ++++++-
 3 files changed, 86 insertions(+), 4 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/klement.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 0b2fd0b48..04309465b 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -20,6 +20,7 @@ include("falsi.jl")
 include("raphson.jl")
 include("ad.jl")
 include("broyden.jl")
+include("klement.jl")
 
 import SnoopPrecompile
 
@@ -46,6 +47,6 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
 end end
 
 # DiffEq styled algorithms
-export Bisection, Broyden, Falsi, SimpleNewtonRaphson
+export Bisection, Broyden, Falsi, Klement, SimpleNewtonRaphson
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
new file mode 100644
index 000000000..86ecd7ef8
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -0,0 +1,65 @@
+"""
+```julia
+Klement()
+```
+
+A low-overhead implementation of [Klement](https://jatm.com.br/jatm/article/view/373).
+This method is non-allocating on scalar and static array problems.
+
+
+"""
+struct Klement <: AbstractSimpleNonlinearSolveAlgorithm end
+
+function SciMLBase.solve(prob::NonlinearProblem,
+                         alg::Klement, args...; abstol = nothing,
+                         reltol = nothing,
+                         maxiters = 1000, kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    x = float(prob.u0)
+    fₙ = f(x)
+    T = eltype(x)
+    J = ArrayInterfaceCore.zeromatrix(x) + I
+
+    if SciMLBase.isinplace(prob)
+        error("Klement currently only supports out-of-place nonlinear problems")
+    end
+
+    atol = abstol !== nothing ? abstol :
+           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
+    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+
+    xₙ = x
+    xₙ₋₁ = x
+    fₙ₋₁ = fₙ
+    for _ in 1:maxiters
+
+        xₙ = xₙ₋₁ - inv(J) * fₙ₋₁
+        fₙ = f(xₙ)
+        Δxₙ = xₙ - xₙ₋₁
+        Δfₙ = fₙ - fₙ₋₁
+
+        # Prevent division by 0
+        denominator = max.(J' .^ 2 * Δxₙ .^ 2, 1e-9)
+
+        k = (Δfₙ - J * Δxₙ) ./ denominator
+        J += (k * Δxₙ' .* J) * J
+
+        # Prevent inverting singular matrix
+        if det(J) ≈ 0
+            J = ArrayInterfaceCore.zeromatrix(x) + I
+        end
+
+        iszero(fₙ) &&
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                            retcode = ReturnCode.Success)
+
+        if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                            retcode = ReturnCode.Success)
+        end
+        xₙ₋₁ = xₙ
+        fₙ₋₁ = fₙ
+    end
+
+    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 6a5f69b3f..648c16fb4 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -35,13 +35,24 @@ sol = benchmark_scalar(sf, csu0)
 @test sol.u * sol.u - 2 < 1e-9
 @test (@ballocated benchmark_scalar(sf, csu0)) == 0
 
+# Klement
+function benchmark_scalar(f, u0)
+    probN = NonlinearProblem{false}(f, u0)
+    sol = (solve(probN, Klement()))
+end
+
+sol = benchmark_scalar(sf, csu0)
+@test sol.retcode === ReturnCode.Success
+@test sol.u * sol.u - 2 < 1e-9
+@test (@ballocated benchmark_scalar(sf, csu0)) == 0
+
 # AD Tests
 using ForwardDiff
 
 # Immutable
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
-for alg in [SimpleNewtonRaphson(), Broyden()]
+for alg in [SimpleNewtonRaphson(), Broyden(), Klement()]
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, tol = 1e-9)
@@ -56,7 +67,7 @@ end
 
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
-for alg in [SimpleNewtonRaphson(), Broyden()]
+for alg in [SimpleNewtonRaphson(), Broyden(), Klement()]
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -97,7 +108,7 @@ for alg in [Bisection(), Falsi()]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
 
-for alg in [SimpleNewtonRaphson(), Broyden()]
+for alg in [SimpleNewtonRaphson(), Broyden(), Klement()]
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -120,6 +131,9 @@ probN = NonlinearProblem(f, u0)
 @test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
 @test solve(probN, Broyden(); immutable = false).u[end] ≈ sqrt(2.0)
 
+@test solve(probN, Klement()).u[end] ≈ sqrt(2.0)
+@test solve(probN, Klement(); immutable = false).u[end] ≈ sqrt(2.0)
+
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol
     f = (u, p) -> u .* u .- 2.0
@@ -131,6 +145,8 @@ for u0 in [1.0, [1, 1.0]]
     @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol
 
     @test solve(probN, Broyden()).u ≈ sol
+
+    @test solve(probN, Klement()).u ≈ sol
 end
 
 # Bisection Tests

From ac619dc25f8599d39240c84f9a00c98da50747a8 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Tue, 3 Jan 2023 17:36:18 +0100
Subject: [PATCH 025/375] format fix

---
 lib/SimpleNonlinearSolve/src/klement.jl | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 86ecd7ef8..603516751 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -5,8 +5,6 @@ Klement()
 
 A low-overhead implementation of [Klement](https://jatm.com.br/jatm/article/view/373).
 This method is non-allocating on scalar and static array problems.
-
-
 """
 struct Klement <: AbstractSimpleNonlinearSolveAlgorithm end
 
@@ -32,7 +30,6 @@ function SciMLBase.solve(prob::NonlinearProblem,
     xₙ₋₁ = x
     fₙ₋₁ = fₙ
     for _ in 1:maxiters
-
         xₙ = xₙ₋₁ - inv(J) * fₙ₋₁
         fₙ = f(xₙ)
         Δxₙ = xₙ - xₙ₋₁

From 433e7c7bc712422c0f2423a5c277e1687081ce20 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Wed, 4 Jan 2023 12:15:57 +0100
Subject: [PATCH 026/375] change singularity handeling, and init of J

---
 lib/SimpleNonlinearSolve/src/klement.jl | 29 ++++++++++++++-----------
 lib/SimpleNonlinearSolve/src/utils.jl   | 10 +++++++++
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 603516751..77e0fa14f 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -16,7 +16,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     x = float(prob.u0)
     fₙ = f(x)
     T = eltype(x)
-    J = ArrayInterfaceCore.zeromatrix(x) + I
+    J = init_J(x)
 
     if SciMLBase.isinplace(prob)
         error("Klement currently only supports out-of-place nonlinear problems")
@@ -30,8 +30,19 @@ function SciMLBase.solve(prob::NonlinearProblem,
     xₙ₋₁ = x
     fₙ₋₁ = fₙ
     for _ in 1:maxiters
-        xₙ = xₙ₋₁ - inv(J) * fₙ₋₁
+        tmp = J \ fₙ₋₁
+        xₙ = xₙ₋₁ - tmp
         fₙ = f(xₙ)
+
+        iszero(fₙ) &&
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                            retcode = ReturnCode.Success)
+
+        if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                            retcode = ReturnCode.Success)
+        end
+
         Δxₙ = xₙ - xₙ₋₁
         Δfₙ = fₙ - fₙ₋₁
 
@@ -41,19 +52,11 @@ function SciMLBase.solve(prob::NonlinearProblem,
         k = (Δfₙ - J * Δxₙ) ./ denominator
         J += (k * Δxₙ' .* J) * J
 
-        # Prevent inverting singular matrix
-        if det(J) ≈ 0
-            J = ArrayInterfaceCore.zeromatrix(x) + I
+        # Singularity test
+        if cond(J) > 1e9
+            J = init_J(xₙ)
         end
 
-        iszero(fₙ) &&
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                            retcode = ReturnCode.Success)
-
-        if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                            retcode = ReturnCode.Success)
-        end
         xₙ₋₁ = xₙ
         fₙ₋₁ = fₙ
     end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 25f6ba358..a4d86e2f8 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -33,3 +33,13 @@ value_derivative(f::F, x::AbstractArray) where {F} = f(x), ForwardDiff.jacobian(
 value(x) = x
 value(x::Dual) = ForwardDiff.value(x)
 value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
+
+function init_J(x)
+    J = ArrayInterfaceCore.zeromatrix(x)
+    if ismutable(x)
+        J[diagind(J)] .= one(eltype(x))
+    else
+        J += I
+    end
+    return J
+end

From 27f5502d02313835dedd42ec74f81c43ca0a67b0 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Wed, 4 Jan 2023 18:57:58 +0100
Subject: [PATCH 027/375] using lu-factorization

---
 lib/SimpleNonlinearSolve/src/broyden.jl |  2 +-
 lib/SimpleNonlinearSolve/src/klement.jl | 94 ++++++++++++++++++-------
 2 files changed, 69 insertions(+), 27 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 49f53950a..06f923de2 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -16,7 +16,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     x = float(prob.u0)
     fₙ = f(x)
     T = eltype(x)
-    J⁻¹ = ArrayInterfaceCore.zeromatrix(x) + I
+    J⁻¹ = init_J(x)
 
     if SciMLBase.isinplace(prob)
         error("Broyden currently only supports out-of-place nonlinear problems")
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 77e0fa14f..1209e9590 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -4,7 +4,7 @@ Klement()
 ```
 
 A low-overhead implementation of [Klement](https://jatm.com.br/jatm/article/view/373).
-This method is non-allocating on scalar and static array problems.
+This method is non-allocating on scalar problems.
 """
 struct Klement <: AbstractSimpleNonlinearSolveAlgorithm end
 
@@ -16,7 +16,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     x = float(prob.u0)
     fₙ = f(x)
     T = eltype(x)
-    J = init_J(x)
+    singular_tol = 1e-9
 
     if SciMLBase.isinplace(prob)
         error("Klement currently only supports out-of-place nonlinear problems")
@@ -29,36 +29,78 @@ function SciMLBase.solve(prob::NonlinearProblem,
     xₙ = x
     xₙ₋₁ = x
     fₙ₋₁ = fₙ
-    for _ in 1:maxiters
-        tmp = J \ fₙ₋₁
-        xₙ = xₙ₋₁ - tmp
-        fₙ = f(xₙ)
-
-        iszero(fₙ) &&
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                            retcode = ReturnCode.Success)
-
-        if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                            retcode = ReturnCode.Success)
-        end
 
-        Δxₙ = xₙ - xₙ₋₁
-        Δfₙ = fₙ - fₙ₋₁
+    # x is scalar
+    if isa(x, Number)
+        J = 1.0
+        for _ in 1:maxiters
+
+            xₙ = xₙ₋₁ - fₙ₋₁/J
+            fₙ = f(xₙ)
+
+            iszero(fₙ) &&
+                return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                                retcode = ReturnCode.Success)
+
+            if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
+                return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                                retcode = ReturnCode.Success)
+            end
 
-        # Prevent division by 0
-        denominator = max.(J' .^ 2 * Δxₙ .^ 2, 1e-9)
+            Δxₙ = xₙ - xₙ₋₁
+            Δfₙ = fₙ - fₙ₋₁
 
-        k = (Δfₙ - J * Δxₙ) ./ denominator
-        J += (k * Δxₙ' .* J) * J
+            # Prevent division by 0
+            denominator = max(J ^ 2 * Δxₙ ^ 2, 1e-9)
 
-        # Singularity test
-        if cond(J) > 1e9
-            J = init_J(xₙ)
+            k = (Δfₙ - J * Δxₙ) / denominator
+            J += (k * Δxₙ * J) * J
+
+            # Singularity test
+            if J < singular_tol
+                J = 1.0
+            end
+
+            xₙ₋₁ = xₙ
+            fₙ₋₁ = fₙ
         end
+    # x is a vector
+    else
+        J = init_J(x)
+        F = lu(J, check = false)
+        for _ in 1:maxiters
+            tmp = F \ fₙ₋₁
+            xₙ = xₙ₋₁ - tmp
+            fₙ = f(xₙ)
+
+            iszero(fₙ) &&
+                return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                                retcode = ReturnCode.Success)
 
-        xₙ₋₁ = xₙ
-        fₙ₋₁ = fₙ
+            if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
+                return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                                retcode = ReturnCode.Success)
+            end
+
+            Δxₙ = xₙ - xₙ₋₁
+            Δfₙ = fₙ - fₙ₋₁
+
+            # Prevent division by 0
+            denominator = max.(J' .^ 2 * Δxₙ .^ 2, 1e-9)
+
+            k = (Δfₙ - J * Δxₙ) ./ denominator
+            J += (k * Δxₙ' .* J) * J
+            F = lu(J, check = false)
+
+            # Singularity test
+            if any(abs.(F.U[diagind(F.U)]) .< singular_tol)
+                J = init_J(xₙ)
+                F = lu(J, check = false)
+            end
+
+            xₙ₋₁ = xₙ
+            fₙ₋₁ = fₙ
+        end
     end
 
     return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)

From 17bb068288b5fd47fed2ff12493451e471e8c701 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Wed, 4 Jan 2023 19:41:03 +0100
Subject: [PATCH 028/375] formating

---
 lib/SimpleNonlinearSolve/src/klement.jl | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 1209e9590..4804e0922 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -34,8 +34,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     if isa(x, Number)
         J = 1.0
         for _ in 1:maxiters
-
-            xₙ = xₙ₋₁ - fₙ₋₁/J
+            xₙ = xₙ₋₁ - fₙ₋₁ / J
             fₙ = f(xₙ)
 
             iszero(fₙ) &&
@@ -51,7 +50,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
             Δfₙ = fₙ - fₙ₋₁
 
             # Prevent division by 0
-            denominator = max(J ^ 2 * Δxₙ ^ 2, 1e-9)
+            denominator = max(J^2 * Δxₙ^2, 1e-9)
 
             k = (Δfₙ - J * Δxₙ) / denominator
             J += (k * Δxₙ * J) * J
@@ -64,7 +63,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
             xₙ₋₁ = xₙ
             fₙ₋₁ = fₙ
         end
-    # x is a vector
+        # x is a vector
     else
         J = init_J(x)
         F = lu(J, check = false)

From d71882cf01b419d61fc5849103b5b74a27f5d5d6 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 4 Jan 2023 15:25:04 -0500
Subject: [PATCH 029/375] Update src/klement.jl

---
 lib/SimpleNonlinearSolve/src/klement.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 4804e0922..04b5787b8 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -31,7 +31,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     fₙ₋₁ = fₙ
 
     # x is scalar
-    if isa(x, Number)
+    if x isa Number
         J = 1.0
         for _ in 1:maxiters
             xₙ = xₙ₋₁ - fₙ₋₁ / J

From 0cb28987e7671c10427e5e7eb888605c8e2400cf Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Wed, 4 Jan 2023 22:05:44 +0100
Subject: [PATCH 030/375] moving the rows around => 1 less lu-factorization

---
 lib/SimpleNonlinearSolve/src/klement.jl | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 4804e0922..1d8b8afc7 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -66,8 +66,15 @@ function SciMLBase.solve(prob::NonlinearProblem,
         # x is a vector
     else
         J = init_J(x)
-        F = lu(J, check = false)
         for _ in 1:maxiters
+            F = lu(J, check = false)
+
+            # Singularity test
+            if any(abs.(F.U[diagind(F.U)]) .< singular_tol)
+                J = init_J(xₙ)
+                F = lu(J, check = false)
+            end
+
             tmp = F \ fₙ₋₁
             xₙ = xₙ₋₁ - tmp
             fₙ = f(xₙ)
@@ -89,13 +96,6 @@ function SciMLBase.solve(prob::NonlinearProblem,
 
             k = (Δfₙ - J * Δxₙ) ./ denominator
             J += (k * Δxₙ' .* J) * J
-            F = lu(J, check = false)
-
-            # Singularity test
-            if any(abs.(F.U[diagind(F.U)]) .< singular_tol)
-                J = init_J(xₙ)
-                F = lu(J, check = false)
-            end
 
             xₙ₋₁ = xₙ
             fₙ₋₁ = fₙ

From 1595d0ee77bd4e0f43eda0fac8d281b1a64514bb Mon Sep 17 00:00:00 2001
From: Arno Strouwen <arno.strouwen@telenet.be>
Date: Wed, 4 Jan 2023 23:10:54 +0100
Subject: [PATCH 031/375] [skip ci] spelling

---
 lib/SimpleNonlinearSolve/src/bisection.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index e90f28f9e..4e2b232e6 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -6,9 +6,9 @@ A common bisection method.
 ### Keyword Arguments
 
 - `exact_left`: whether to enforce whether the left side of the interval must be exactly
-  zero for the returned result. Defualts to false.
+  zero for the returned result. Defaults to false.
 - `exact_right`: whether to enforce whether the right side of the interval must be exactly
-  zero for the returned result. Defualts to false.
+  zero for the returned result. Defaults to false.
 """
 struct Bisection <: AbstractBracketingAlgorithm
     exact_left::Bool

From f419c140cbbb6f9d9d49db9a8e68e6aff685fabe Mon Sep 17 00:00:00 2001
From: Arno Strouwen <arno.strouwen@telenet.be>
Date: Wed, 4 Jan 2023 23:12:52 +0100
Subject: [PATCH 032/375] [skip ci] spelling

---
 lib/SimpleNonlinearSolve/src/raphson.jl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 47ae6eaf8..711f7766e 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -10,18 +10,18 @@ and static array problems.
 !!! note
 
     As part of the decreased overhead, this method omits some of the higher level error
-    catching of the other methods. Thus to see better error messages, use one of the other
+    catching of the other methods. Thus, to see better error messages, use one of the other
     methods like `NewtonRaphson`
 
 ### Keyword Arguments
 
 - `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
-  system. This allows for multiple derivative columns to be computed simultaniously,
+  system. This allows for multiple derivative columns to be computed simultaneously,
   improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
   default chunk size mechanism. For more details, see the documentation for
   [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
 - `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
-  Note that this argument is ignored if an analytical Jacobian is passed as that will be
+  Note that this argument is ignored if an analytical Jacobian is passed; as that will be
   used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
   If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
 - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to

From 2a8b42cf8a385d5ed3bb80e418138c6c7bfc8ef7 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Thu, 5 Jan 2023 14:06:16 +0100
Subject: [PATCH 033/375] Missed to precompile Klement

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 04309465b..a91b05719 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -26,7 +26,7 @@ import SnoopPrecompile
 
 SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-    for alg in (SimpleNewtonRaphson, Broyden)
+    for alg in (SimpleNewtonRaphson, Broyden, Klement)
         solve(prob_no_brack, alg(), tol = T(1e-2))
     end
 

From 175d656f5133d02a1e4fc43ce65d5363e80023a8 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Tue, 10 Jan 2023 20:33:50 +0100
Subject: [PATCH 034/375] Implementation of a trust-region solver

---
 .../src/SimpleNonlinearSolve.jl               |   8 +-
 lib/SimpleNonlinearSolve/src/trustRegion.jl   | 125 ++++++++++++++++++
 lib/SimpleNonlinearSolve/src/utils.jl         |  25 ++++
 lib/SimpleNonlinearSolve/test/basictests.jl   |  28 +++-
 4 files changed, 182 insertions(+), 4 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/trustRegion.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index a91b05719..4937ae119 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -21,6 +21,7 @@ include("raphson.jl")
 include("ad.jl")
 include("broyden.jl")
 include("klement.jl")
+include("trustRegion.jl")
 
 import SnoopPrecompile
 
@@ -30,6 +31,10 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
         solve(prob_no_brack, alg(), tol = T(1e-2))
     end
 
+    for alg in (TrustRegion(1.0),)
+        solve(prob_no_brack, alg, tol = T(1e-2))
+    end
+
     #=
     for alg in (SimpleNewtonRaphson,)
         for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
@@ -47,6 +52,7 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
 end end
 
 # DiffEq styled algorithms
-export Bisection, Broyden, Falsi, Klement, SimpleNewtonRaphson
+export Bisection, Broyden, Falsi, Klement, SimpleNewtonRaphson,
+       TrustRegion
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
new file mode 100644
index 000000000..1d017a80d
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -0,0 +1,125 @@
+"""
+```julia
+TrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
+                               autodiff = Val{true}(), diff_type = Val{:forward})
+```
+
+A low-overhead implementation of a
+[trust-region](https://optimization.mccormick.northwestern.edu/index.php/Trust-region_methods)
+solver
+
+
+### Keyword Arguments
+- `max_trust_radius`: the maximum radius of the trust region. The step size in the algorithm
+  will change dynamically. However, it will never be greater than the `max_trust_radius`.
+
+### Keyword Arguments
+
+- `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
+  system. This allows for multiple derivative columns to be computed simultaneously,
+  improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
+  default chunk size mechanism. For more details, see the documentation for
+  [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
+- `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
+  Note that this argument is ignored if an analytical Jacobian is passed; as that will be
+  used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
+  If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
+- `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
+  `Val{:forward}` for forward finite differences. For more details on the choices, see the
+  [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+"""
+struct TrustRegion{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
+    max_trust_radius::Number
+    function TrustRegion(max_turst_radius::Number; chunk_size = Val{0}(),
+                                            autodiff = Val{true}(),
+                                            diff_type = Val{:forward})
+        new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
+            SciMLBase._unwrap_val(diff_type)}(max_trust_radius)
+    end
+end
+
+function SciMLBase.solve(prob::NonlinearProblem,
+                         alg::TrustRegion, args...; abstol = nothing,
+                         reltol = nothing,
+                         maxiters = 1000, kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    x = float(prob.u0)
+    T = typeof(x)
+    Δₘₐₓ = float(alg.max_trust_radius)  # The maximum trust region radius.
+    Δ = Δₘₐₓ / 5  # Initial trust region radius.
+    η₁ = 0.1   # Threshold for taking a step.
+    η₂ = 0.25  # Threshold for shrinking the trust region.
+    η₃ = 0.75  # Threshold for expanding the trust region.
+    t₁ = 0.25  # Factor to shrink the trust region with.
+    t₂ = 2.0   # Factor to expand the trust region with.
+
+    if SciMLBase.isinplace(prob)
+        error("TrustRegion currently only supports out-of-place nonlinear problems")
+    end
+
+    atol = abstol !== nothing ? abstol :
+           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
+    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+
+    if alg_autodiff(alg)
+        F, ∇f = value_derivative(f, x)
+    elseif x isa AbstractArray
+        F = f(x)
+        ∇f = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x), F)
+    else
+        F = f(x)
+        ∇f = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x), F)
+    end
+
+    fₖ = 0.5 * norm(F)^2
+    H = ∇f * ∇f
+    g = ∇f * F
+
+    for k in 1:maxiters
+        # Solve the trust region subproblem.
+        δ = dogleg_method(H, g, Δ)
+        xₖ₊₁ = x + δ
+        Fₖ₊₁ = f(xₖ₊₁)
+        fₖ₊₁ = 0.5 * norm(Fₖ₊₁)^2
+
+        # Compute the ratio of the actual to predicted reduction.
+        model = -(δ' * g + 0.5 * δ' * H * δ)
+        r = (fₖ - fₖ₊₁) / model
+
+        # Update the trust region radius.
+        if r < η₂
+            Δ *= t₁
+        if r > η₁
+            if isapprox(x̂, x, atol = atol, rtol = rtol)
+                return SciMLBase.build_solution(prob, alg, x, F;
+                                                retcode = ReturnCode.Success)
+            end
+
+            x = xₖ₊₁
+            F = Fₖ₊₁
+            if alg_autodiff(alg)
+                F, ∇f = value_derivative(f, x)
+            elseif x isa AbstractArray
+                ∇f = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x),
+                                                            F)
+            else
+                ∇f = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg),
+                                                              eltype(x),
+                                                              F)
+            end
+
+            iszero(F) &&
+                return SciMLBase.build_solution(prob, alg, x, F;
+                                                retcode = ReturnCode.Success)
+            # Update the trust region radius.
+            if r > η₃ && norm(δ) ≈ Δ
+                Δ = min(t₂ * Δ, Δₘₐₓ)
+            end
+            fₖ = f̂
+            H = ∇f * ∇f
+            g = ∇f * F
+        end
+    end
+
+    return SciMLBase.build_solution(prob, alg, x, F; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index a4d86e2f8..f3b7de9f6 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -43,3 +43,28 @@ function init_J(x)
     end
     return J
 end
+
+function dogleg_method(H, g, Δ)
+    # Compute the Newton step.
+    δN = -H \ g
+    # Test if the full step is within the trust region.
+    if norm(δN) ≤ Δ
+        return δN
+    end
+
+    # Calcualte Cauchy point, optimum along the steepest descent direction.
+    δsd = -g
+    norm_δsd = norm(δsd)
+    if norm_δsd ≥ Δ
+        return δsd .* Δ / norm_δsd
+    end
+
+    # Find the intersection point on the boundary.
+    δN_δsd = δN - δsd
+    dot_δN_δsd = dot(δN_δsd, δN_δsd)
+    dot_δsd_δN_δsd = dot(δsd, δN_δsd)
+    dot_δsd = dot(δsd, δsd)
+    fact = dot_δsd_δN_δsd^2 - dot_δN_δsd * (dot_δsd - Δ^2)
+    tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd
+    return δsd + tau * δN_δsd
+end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 648c16fb4..9122519f8 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -46,13 +46,24 @@ sol = benchmark_scalar(sf, csu0)
 @test sol.u * sol.u - 2 < 1e-9
 @test (@ballocated benchmark_scalar(sf, csu0)) == 0
 
+# SimpleNewtonRaphsonTrustRegion
+function benchmark_scalar(f, u0)
+    probN = NonlinearProblem{false}(f, u0)
+    sol = (solve(probN, SimpleNewtonRaphsonTrustRegion(1.0)))
+end
+
+sol = benchmark_scalar(sf, csu0)
+@test sol.retcode === ReturnCode.Success
+@test sol.u * sol.u - 2 < 1e-9
+
 # AD Tests
 using ForwardDiff
 
 # Immutable
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
-for alg in [SimpleNewtonRaphson(), Broyden(), Klement()]
+for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
+            SimpleNewtonRaphsonTrustRegion(10.0)]
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, tol = 1e-9)
@@ -67,7 +78,8 @@ end
 
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
-for alg in [SimpleNewtonRaphson(), Broyden(), Klement()]
+for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
+            SimpleNewtonRaphsonTrustRegion(10.0)]
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -108,7 +120,8 @@ for alg in [Bisection(), Falsi()]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
 
-for alg in [SimpleNewtonRaphson(), Broyden(), Klement()]
+for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
+            SimpleNewtonRaphsonTrustRegion(1.0)]
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -128,6 +141,11 @@ probN = NonlinearProblem(f, u0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
 
+@test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0)).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0); immutable = false).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0; autodiff = false)).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0; autodiff = false)).u[end] ≈ sqrt(2.0)
+
 @test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
 @test solve(probN, Broyden(); immutable = false).u[end] ≈ sqrt(2.0)
 
@@ -144,6 +162,10 @@ for u0 in [1.0, [1, 1.0]]
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol
 
+    @test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0)).u ≈ sol
+    @test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0)).u ≈ sol
+    @test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0; autodiff = false)).u ≈ sol
+
     @test solve(probN, Broyden()).u ≈ sol
 
     @test solve(probN, Klement()).u ≈ sol

From bc4ab2d33200eb0c192502ed83aacee088c611d5 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Tue, 10 Jan 2023 21:46:51 +0100
Subject: [PATCH 035/375] bug fix

---
 .../src/SimpleNonlinearSolve.jl               |  2 +-
 lib/SimpleNonlinearSolve/src/trustRegion.jl   | 34 +++++++++++--------
 lib/SimpleNonlinearSolve/test/basictests.jl   | 24 ++++++-------
 3 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 4937ae119..b4c094493 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -31,7 +31,7 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
         solve(prob_no_brack, alg(), tol = T(1e-2))
     end
 
-    for alg in (TrustRegion(1.0),)
+    for alg in (TrustRegion(10.0),)
         solve(prob_no_brack, alg, tol = T(1e-2))
     end
 
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 1d017a80d..cfdf9887d 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -30,9 +30,9 @@ solver
 """
 struct TrustRegion{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     max_trust_radius::Number
-    function TrustRegion(max_turst_radius::Number; chunk_size = Val{0}(),
-                                            autodiff = Val{true}(),
-                                            diff_type = Val{:forward})
+    function TrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
+                         autodiff = Val{true}(),
+                         diff_type = Val{:forward})
         new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
             SciMLBase._unwrap_val(diff_type)}(max_trust_radius)
     end
@@ -46,8 +46,8 @@ function SciMLBase.solve(prob::NonlinearProblem,
     x = float(prob.u0)
     T = typeof(x)
     Δₘₐₓ = float(alg.max_trust_radius)  # The maximum trust region radius.
-    Δ = Δₘₐₓ / 5  # Initial trust region radius.
-    η₁ = 0.1   # Threshold for taking a step.
+    Δ = Δₘₐₓ / 11  # Initial trust region radius.
+    η₁ = 0.0   # Threshold for taking a step.
     η₂ = 0.25  # Threshold for shrinking the trust region.
     η₃ = 0.75  # Threshold for expanding the trust region.
     t₁ = 0.25  # Factor to shrink the trust region with.
@@ -88,38 +88,44 @@ function SciMLBase.solve(prob::NonlinearProblem,
 
         # Update the trust region radius.
         if r < η₂
-            Δ *= t₁
-        if r > η₁
-            if isapprox(x̂, x, atol = atol, rtol = rtol)
+            Δ = t₁ * Δ
+
+            if Δ < 1e-10
                 return SciMLBase.build_solution(prob, alg, x, F;
                                                 retcode = ReturnCode.Success)
             end
-
+        end
+        if r > η₁
+            if isapprox(xₖ₊₁, x, atol = atol, rtol = rtol)
+                return SciMLBase.build_solution(prob, alg, xₖ₊₁, Fₖ₊₁;
+                                                retcode = ReturnCode.Success)
+            end
+            # Take the step.
             x = xₖ₊₁
             F = Fₖ₊₁
             if alg_autodiff(alg)
                 F, ∇f = value_derivative(f, x)
             elseif x isa AbstractArray
                 ∇f = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x),
-                                                            F)
+                                                           F)
             else
                 ∇f = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg),
-                                                              eltype(x),
-                                                              F)
+                                                             eltype(x),
+                                                             F)
             end
 
             iszero(F) &&
                 return SciMLBase.build_solution(prob, alg, x, F;
                                                 retcode = ReturnCode.Success)
+
             # Update the trust region radius.
             if r > η₃ && norm(δ) ≈ Δ
                 Δ = min(t₂ * Δ, Δₘₐₓ)
             end
-            fₖ = f̂
+            fₖ = fₖ₊₁
             H = ∇f * ∇f
             g = ∇f * F
         end
     end
-
     return SciMLBase.build_solution(prob, alg, x, F; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 9122519f8..f30586c22 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -46,10 +46,10 @@ sol = benchmark_scalar(sf, csu0)
 @test sol.u * sol.u - 2 < 1e-9
 @test (@ballocated benchmark_scalar(sf, csu0)) == 0
 
-# SimpleNewtonRaphsonTrustRegion
+# TrustRegion
 function benchmark_scalar(f, u0)
     probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, SimpleNewtonRaphsonTrustRegion(1.0)))
+    sol = (solve(probN, TrustRegion(10.0)))
 end
 
 sol = benchmark_scalar(sf, csu0)
@@ -63,7 +63,7 @@ using ForwardDiff
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
 for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-            SimpleNewtonRaphsonTrustRegion(10.0)]
+            TrustRegion(10.0)]
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, tol = 1e-9)
@@ -79,7 +79,7 @@ end
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
 for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-            SimpleNewtonRaphsonTrustRegion(10.0)]
+            TrustRegion(10.0)]
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -121,7 +121,7 @@ for alg in [Bisection(), Falsi()]
 end
 
 for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-            SimpleNewtonRaphsonTrustRegion(1.0)]
+            TrustRegion(10.0)]
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -141,10 +141,10 @@ probN = NonlinearProblem(f, u0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
 
-@test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0)).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0); immutable = false).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0; autodiff = false)).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0; autodiff = false)).u[end] ≈ sqrt(2.0)
+@test solve(probN, TrustRegion(10.0)).u[end] ≈ sqrt(2.0)
+@test solve(probN, TrustRegion(10.0); immutable = false).u[end] ≈ sqrt(2.0)
+@test solve(probN, TrustRegion(10.0; autodiff = false)).u[end] ≈ sqrt(2.0)
+@test solve(probN, TrustRegion(10.0; autodiff = false)).u[end] ≈ sqrt(2.0)
 
 @test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
 @test solve(probN, Broyden(); immutable = false).u[end] ≈ sqrt(2.0)
@@ -162,9 +162,9 @@ for u0 in [1.0, [1, 1.0]]
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol
 
-    @test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0)).u ≈ sol
-    @test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0)).u ≈ sol
-    @test solve(probN, SimpleNewtonRaphsonTrustRegion(1.0; autodiff = false)).u ≈ sol
+    @test solve(probN, TrustRegion(10.0)).u ≈ sol
+    @test solve(probN, TrustRegion(10.0)).u ≈ sol
+    @test solve(probN, TrustRegion(10.0; autodiff = false)).u ≈ sol
 
     @test solve(probN, Broyden()).u ≈ sol
 

From 28aebe68542271d374db284cf7ac8f2ef64facff Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Tue, 10 Jan 2023 21:49:09 +0100
Subject: [PATCH 036/375] update of parameter

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index cfdf9887d..781085cc1 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -47,7 +47,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     T = typeof(x)
     Δₘₐₓ = float(alg.max_trust_radius)  # The maximum trust region radius.
     Δ = Δₘₐₓ / 11  # Initial trust region radius.
-    η₁ = 0.0   # Threshold for taking a step.
+    η₁ = 0.1   # Threshold for taking a step.
     η₂ = 0.25  # Threshold for shrinking the trust region.
     η₃ = 0.75  # Threshold for expanding the trust region.
     t₁ = 0.25  # Factor to shrink the trust region with.

From 883aeacd3ff5276c4b662b315180f8cbea8a6f48 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Tue, 10 Jan 2023 21:51:08 +0100
Subject: [PATCH 037/375] format

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index b4c094493..5f269fe12 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -52,7 +52,6 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
 end end
 
 # DiffEq styled algorithms
-export Bisection, Broyden, Falsi, Klement, SimpleNewtonRaphson,
-       TrustRegion
+export Bisection, Broyden, Falsi, Klement, SimpleNewtonRaphson, TrustRegion
 
 end # module

From c555f51833cde8c3d7113b88096068d9045118a9 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Tue, 10 Jan 2023 21:58:40 +0100
Subject: [PATCH 038/375] Implemented max allowed shrink of trust-region

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 781085cc1..dd91353e6 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -74,6 +74,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     fₖ = 0.5 * norm(F)^2
     H = ∇f * ∇f
     g = ∇f * F
+    counter = 0
 
     for k in 1:maxiters
         # Solve the trust region subproblem.
@@ -89,11 +90,13 @@ function SciMLBase.solve(prob::NonlinearProblem,
         # Update the trust region radius.
         if r < η₂
             Δ = t₁ * Δ
-
-            if Δ < 1e-10
+            counter += 1
+            if counter > 32
                 return SciMLBase.build_solution(prob, alg, x, F;
                                                 retcode = ReturnCode.Success)
             end
+        else
+            counter = 0
         end
         if r > η₁
             if isapprox(xₖ₊₁, x, atol = atol, rtol = rtol)

From 36a3f3cc54819fab66975a23f6e33155337a8a9a Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Wed, 11 Jan 2023 09:11:38 +0100
Subject: [PATCH 039/375] Made kwargs of parameters

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 72 ++++++++++++++++-----
 lib/SimpleNonlinearSolve/test/basictests.jl | 57 +++++++++++++++-
 2 files changed, 110 insertions(+), 19 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index dd91353e6..efc30969f 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -8,8 +8,7 @@ A low-overhead implementation of a
 [trust-region](https://optimization.mccormick.northwestern.edu/index.php/Trust-region_methods)
 solver
 
-
-### Keyword Arguments
+### Arguments
 - `max_trust_radius`: the maximum radius of the trust region. The step size in the algorithm
   will change dynamically. However, it will never be greater than the `max_trust_radius`.
 
@@ -27,14 +26,54 @@ solver
 - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
   `Val{:forward}` for forward finite differences. For more details on the choices, see the
   [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+- `initial_trust_radius`: the initial trust region radius. Defaults to
+  `max_trust_radius / 11`.
+- `step_threshold`: the threshold for taking a step. In every iteration, the threshold is
+  compared with a value `r`, which is the actual reduction in the objective function divided
+  by the predicted reduction. If `step_threshold > r` the model is not a good approximation,
+  and the step is rejected. Defaults to `0.1`. For more details, see
+  [Trust-region methods](https://optimization.mccormick.northwestern.edu/index.php/Trust-region_methods)
+- `shrink_threshold`: the threshold for shrinking the trust region radius. In every
+  iteration, the threshold is compared with a value `r` which is the actual reduction in the
+  objective function divided by the predicted reduction. If `shrink_threshold > r` the trust
+  region radius is shrunk by `shrink_factor`. Defaults to `0.25`. For more details, see
+  [Trust-region methods](https://optimization.mccormick.northwestern.edu/index.php/Trust-region_methods)
+- `expand_threshold`: the threshold for expanding the trust region radius. If a step is
+  taken, i.e `step_threshold < r` (with `r` defined in `shrink_threshold`), a check is also
+  made to see if `expand_threshold < r`. If that is true, the trust region radius is
+  expanded by `expand_factor`. Defaults to `0.75`.
+- `shrink_factor`: the factor to shrink the trust region radius with if
+  `shrink_threshold > r` (with `r` defined in `shrink_threshold`). Defaults to `0.25`.
+- `expand_factor`: the factor to expand the trust region radius with if
+  `expand_threshold < r` (with `r` defined in `shrink_threshold`). Defaults to `2.0`.
+- `max_shrink_times`: the maximum number of times to shrink the trust region radius in a
+  row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`.
 """
 struct TrustRegion{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     max_trust_radius::Number
+    initial_trust_radius::Number
+    step_threshold::Number
+    shrink_threshold::Number
+    expand_threshold::Number
+    shrink_factor::Number
+    expand_factor::Number
+    max_shrink_times::Int
     function TrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
                          autodiff = Val{true}(),
-                         diff_type = Val{:forward})
+                         diff_type = Val{:forward},
+                         initial_trust_radius::Number = max_trust_radius / 11,
+                         step_threshold::Number = 0.1,
+                         shrink_threshold::Number = 0.25,
+                         expand_threshold::Number = 0.75,
+                         shrink_factor::Number = 0.25,
+                         expand_factor::Number = 2.0,
+                         max_shrink_times::Int = 32)
         new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
-            SciMLBase._unwrap_val(diff_type)}(max_trust_radius)
+            SciMLBase._unwrap_val(diff_type)}(max_trust_radius, initial_trust_radius,
+                                              step_threshold,
+                                              shrink_threshold, expand_threshold,
+                                              shrink_factor,
+                                              expand_factor, max_shrink_times)
     end
 end
 
@@ -45,13 +84,14 @@ function SciMLBase.solve(prob::NonlinearProblem,
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     T = typeof(x)
-    Δₘₐₓ = float(alg.max_trust_radius)  # The maximum trust region radius.
-    Δ = Δₘₐₓ / 11  # Initial trust region radius.
-    η₁ = 0.1   # Threshold for taking a step.
-    η₂ = 0.25  # Threshold for shrinking the trust region.
-    η₃ = 0.75  # Threshold for expanding the trust region.
-    t₁ = 0.25  # Factor to shrink the trust region with.
-    t₂ = 2.0   # Factor to expand the trust region with.
+    Δₘₐₓ = float(alg.max_trust_radius)
+    Δ = float(alg.initial_trust_radius)
+    η₁ = float(alg.step_threshold)
+    η₂ = float(alg.shrink_threshold)
+    η₃ = float(alg.expand_threshold)
+    t₁ = float(alg.shrink_factor)
+    t₂ = float(alg.expand_factor)
+    max_shrink_times = alg.max_shrink_times
 
     if SciMLBase.isinplace(prob)
         error("TrustRegion currently only supports out-of-place nonlinear problems")
@@ -74,7 +114,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     fₖ = 0.5 * norm(F)^2
     H = ∇f * ∇f
     g = ∇f * F
-    counter = 0
+    shrink_counter = 0
 
     for k in 1:maxiters
         # Solve the trust region subproblem.
@@ -85,18 +125,18 @@ function SciMLBase.solve(prob::NonlinearProblem,
 
         # Compute the ratio of the actual to predicted reduction.
         model = -(δ' * g + 0.5 * δ' * H * δ)
-        r = (fₖ - fₖ₊₁) / model
+        r = model \ (fₖ - fₖ₊₁)
 
         # Update the trust region radius.
         if r < η₂
             Δ = t₁ * Δ
-            counter += 1
-            if counter > 32
+            shrink_counter += 1
+            if shrink_counter > max_shrink_times
                 return SciMLBase.build_solution(prob, alg, x, F;
                                                 retcode = ReturnCode.Success)
             end
         else
-            counter = 0
+            shrink_counter = 0
         end
         if r > η₁
             if isapprox(xₖ₊₁, x, atol = atol, rtol = rtol)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index f30586c22..d2a030253 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -63,7 +63,7 @@ using ForwardDiff
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
 for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-            TrustRegion(10.0)]
+    TrustRegion(10.0)]
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, tol = 1e-9)
@@ -79,7 +79,7 @@ end
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
 for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-            TrustRegion(10.0)]
+    TrustRegion(10.0)]
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -121,7 +121,7 @@ for alg in [Bisection(), Falsi()]
 end
 
 for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-            TrustRegion(10.0)]
+    TrustRegion(10.0)]
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -207,3 +207,54 @@ sol = solve(probB, Bisection(; exact_left = true, exact_right = true); immutable
 @test f(nextfloat(sol.left), nothing) >= 0.0
 @test f(sol.right, nothing) >= 0.0
 @test f(prevfloat(sol.right), nothing) <= 0.0
+
+# Test that `TrustRegion` passes a test that `SimpleNewtonRaphson` fails on.
+u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
+global g, f
+f = (u, p) -> 0.010000000000000002 .+
+              10.000000000000002 ./ (1 .+
+               (0.21640425613334457 .+
+                216.40425613334457 ./ (1 .+
+                 (0.21640425613334457 .+
+                  216.40425613334457 ./
+                  (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
+              0.0011552453009332421u
+.-p
+g = function (p)
+    probN = NonlinearProblem{false}(f, u0, p)
+    sol = solve(probN, TrustRegion(100.0))
+    return sol.u
+end
+p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+u = g(p)
+f(u, p)
+@test all(f(u, p) .< 1e-10)
+
+# Test kwars in `TrustRegion`
+max_trust_radius = [10.0, 100.0, 1000.0]
+initial_trust_radius = [10.0, 1.0, 0.1]
+step_threshold = [0.0, 0.01, 0.25]
+shrink_threshold = [0.25, 0.3, 0.5]
+expand_threshold = [0.5, 0.8, 0.9]
+shrink_factor = [0.1, 0.3, 0.5]
+expand_factor = [1.5, 2.0, 3.0]
+max_shrink_times = [10, 20, 30]
+
+list_of_options = zip(max_trust_radius, initial_trust_radius, step_threshold,
+                      shrink_threshold, expand_threshold, shrink_factor,
+                      expand_factor, max_shrink_times)
+for options in list_of_options
+    local probN, sol, alg
+    alg = TrustRegion(options[1];
+                      initial_trust_radius = options[2],
+                      step_threshold = options[3],
+                      shrink_threshold = options[4],
+                      expand_threshold = options[5],
+                      shrink_factor = options[6],
+                      expand_factor = options[7],
+                      max_shrink_times = options[8])
+
+    probN = NonlinearProblem(f, u0, p)
+    sol = solve(probN, alg)
+    @test all(f(u, p) .< 1e-10)
+end

From d163535fa32ccc9e816fd509dcc8c2c7ed221b1a Mon Sep 17 00:00:00 2001
From: Wonseok Shin <wsshin97@gmail.com>
Date: Sat, 14 Jan 2023 15:14:25 -0500
Subject: [PATCH 040/375] Remove unnecessary @show

---
 lib/SimpleNonlinearSolve/src/raphson.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 711f7766e..52888de99 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -80,6 +80,5 @@ function SciMLBase.solve(prob::NonlinearProblem,
         xo = x
     end
 
-    @show x, fx
     return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end

From 85bc0d7c5d4432ad8e01856a195926e94c907ec2 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 14 Jan 2023 15:23:00 -0500
Subject: [PATCH 041/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 3f4c0d749..4dcd425ba 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.4"
+version = "0.1.5"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"

From b47d1d8c4029b35dc3694349a2d1196733af90ee Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 16 Jan 2023 19:35:03 -0500
Subject: [PATCH 042/375] Use DiffEqBase high level handling

---
 lib/SimpleNonlinearSolve/Project.toml                | 2 ++
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 7 ++++---
 lib/SimpleNonlinearSolve/src/broyden.jl              | 2 +-
 lib/SimpleNonlinearSolve/src/klement.jl              | 2 +-
 lib/SimpleNonlinearSolve/src/raphson.jl              | 2 +-
 lib/SimpleNonlinearSolve/src/trustRegion.jl          | 2 +-
 6 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 4dcd425ba..1afc7746f 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -5,6 +5,7 @@ version = "0.1.5"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
+DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -15,6 +16,7 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [compat]
 ArrayInterfaceCore = "0.1.1"
+DiffEqBase = "6.115"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 Reexport = "0.2, 1"
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 5f269fe12..dbe856365 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -6,6 +6,7 @@ using ForwardDiff: Dual
 using StaticArraysCore
 using LinearAlgebra
 import ArrayInterfaceCore
+using DiffEqBase
 
 @reexport using SciMLBase
 
@@ -28,11 +29,11 @@ import SnoopPrecompile
 SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
     for alg in (SimpleNewtonRaphson, Broyden, Klement)
-        solve(prob_no_brack, alg(), tol = T(1e-2))
+        solve(prob_no_brack, alg(), abstol = T(1e-2))
     end
 
     for alg in (TrustRegion(10.0),)
-        solve(prob_no_brack, alg, tol = T(1e-2))
+        solve(prob_no_brack, alg, abstol = T(1e-2))
     end
 
     #=
@@ -47,7 +48,7 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
 
     prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p, T.((0.0, 2.0)), T(2))
     for alg in (Bisection, Falsi)
-        solve(prob_brack, alg(), tol = T(1e-2))
+        solve(prob_brack, alg(), abstol = T(1e-2))
     end
 end end
 
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 06f923de2..c51e5d1db 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -8,7 +8,7 @@ and static array problems.
 """
 struct Broyden <: AbstractSimpleNonlinearSolveAlgorithm end
 
-function SciMLBase.solve(prob::NonlinearProblem,
+function SciMLBase.__solve(prob::NonlinearProblem,
                          alg::Broyden, args...; abstol = nothing,
                          reltol = nothing,
                          maxiters = 1000, kwargs...)
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 9e2821dcb..bc07d664b 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -8,7 +8,7 @@ This method is non-allocating on scalar problems.
 """
 struct Klement <: AbstractSimpleNonlinearSolveAlgorithm end
 
-function SciMLBase.solve(prob::NonlinearProblem,
+function SciMLBase.__solve(prob::NonlinearProblem,
                          alg::Klement, args...; abstol = nothing,
                          reltol = nothing,
                          maxiters = 1000, kwargs...)
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 52888de99..4ccc4fbbd 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -36,7 +36,7 @@ struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     end
 end
 
-function SciMLBase.solve(prob::NonlinearProblem,
+function SciMLBase.__solve(prob::NonlinearProblem,
                          alg::SimpleNewtonRaphson, args...; abstol = nothing,
                          reltol = nothing,
                          maxiters = 1000, kwargs...)
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index efc30969f..7db6f460f 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -77,7 +77,7 @@ struct TrustRegion{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     end
 end
 
-function SciMLBase.solve(prob::NonlinearProblem,
+function SciMLBase.__solve(prob::NonlinearProblem,
                          alg::TrustRegion, args...; abstol = nothing,
                          reltol = nothing,
                          maxiters = 1000, kwargs...)

From 6fbef92b8afd445885f5c132f069364648f850a3 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 16 Jan 2023 20:14:11 -0500
Subject: [PATCH 043/375] fix bound

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 1afc7746f..3095dabff 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -16,7 +16,7 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [compat]
 ArrayInterfaceCore = "0.1.1"
-DiffEqBase = "6.115"
+DiffEqBase = "6.114"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 Reexport = "0.2, 1"

From 96b2e2e57eaab88f0aaf6b4f4ba8f18bfe40f54a Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 16 Jan 2023 20:16:10 -0500
Subject: [PATCH 044/375] format

---
 lib/SimpleNonlinearSolve/src/broyden.jl     | 6 +++---
 lib/SimpleNonlinearSolve/src/klement.jl     | 6 +++---
 lib/SimpleNonlinearSolve/src/raphson.jl     | 6 +++---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 6 +++---
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index c51e5d1db..a05caa00a 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -9,9 +9,9 @@ and static array problems.
 struct Broyden <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-                         alg::Broyden, args...; abstol = nothing,
-                         reltol = nothing,
-                         maxiters = 1000, kwargs...)
+                           alg::Broyden, args...; abstol = nothing,
+                           reltol = nothing,
+                           maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fₙ = f(x)
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index bc07d664b..4d59377df 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -9,9 +9,9 @@ This method is non-allocating on scalar problems.
 struct Klement <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-                         alg::Klement, args...; abstol = nothing,
-                         reltol = nothing,
-                         maxiters = 1000, kwargs...)
+                           alg::Klement, args...; abstol = nothing,
+                           reltol = nothing,
+                           maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fₙ = f(x)
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 4ccc4fbbd..57d66b005 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -37,9 +37,9 @@ struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-                         alg::SimpleNewtonRaphson, args...; abstol = nothing,
-                         reltol = nothing,
-                         maxiters = 1000, kwargs...)
+                           alg::SimpleNewtonRaphson, args...; abstol = nothing,
+                           reltol = nothing,
+                           maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fx = float(prob.u0)
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 7db6f460f..289987b31 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -78,9 +78,9 @@ struct TrustRegion{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-                         alg::TrustRegion, args...; abstol = nothing,
-                         reltol = nothing,
-                         maxiters = 1000, kwargs...)
+                           alg::TrustRegion, args...; abstol = nothing,
+                           reltol = nothing,
+                           maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     T = typeof(x)

From fd87a491162b69231909e9e1248d69b975aa2885 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 16 Jan 2023 20:54:57 -0500
Subject: [PATCH 045/375] fix tests

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index d2a030253..5b0c2f396 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -66,7 +66,7 @@ for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
     TrustRegion(10.0)]
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
-        sol = solve(probN, alg, tol = 1e-9)
+        sol = solve(probN, alg, abstol = 1e-9)
         return sol.u[end]
     end
 
@@ -137,20 +137,11 @@ f, u0 = (u, p) -> u .* u .- 2.0, @SVector[1.0, 1.0]
 probN = NonlinearProblem(f, u0)
 
 @test solve(probN, SimpleNewtonRaphson()).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleNewtonRaphson(); immutable = false).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
-
 @test solve(probN, TrustRegion(10.0)).u[end] ≈ sqrt(2.0)
-@test solve(probN, TrustRegion(10.0); immutable = false).u[end] ≈ sqrt(2.0)
-@test solve(probN, TrustRegion(10.0; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, TrustRegion(10.0; autodiff = false)).u[end] ≈ sqrt(2.0)
-
 @test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
-@test solve(probN, Broyden(); immutable = false).u[end] ≈ sqrt(2.0)
-
 @test solve(probN, Klement()).u[end] ≈ sqrt(2.0)
-@test solve(probN, Klement(); immutable = false).u[end] ≈ sqrt(2.0)
 
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol

From 604b5b71ec8dbd14f401646c709212ddfbc39848 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 16 Jan 2023 21:21:40 -0500
Subject: [PATCH 046/375] non-allocating on v1.7+

ugh v1.6 is now getting old lol
---
 lib/SimpleNonlinearSolve/test/basictests.jl | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 5b0c2f396..cd960b664 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -22,7 +22,9 @@ sol = benchmark_scalar(sf, csu0)
 @test sol.retcode === ReturnCode.Success
 @test sol.u * sol.u - 2 < 1e-9
 
-@test (@ballocated benchmark_scalar(sf, csu0)) == 0
+if VERSION >= v"1.7"
+    @test (@ballocated benchmark_scalar(sf, csu0)) == 0
+end
 
 # Broyden
 function benchmark_scalar(f, u0)
@@ -33,7 +35,9 @@ end
 sol = benchmark_scalar(sf, csu0)
 @test sol.retcode === ReturnCode.Success
 @test sol.u * sol.u - 2 < 1e-9
-@test (@ballocated benchmark_scalar(sf, csu0)) == 0
+if VERSION >= v"1.7"
+    @test (@ballocated benchmark_scalar(sf, csu0)) == 0
+end
 
 # Klement
 function benchmark_scalar(f, u0)
@@ -44,7 +48,9 @@ end
 sol = benchmark_scalar(sf, csu0)
 @test sol.retcode === ReturnCode.Success
 @test sol.u * sol.u - 2 < 1e-9
-@test (@ballocated benchmark_scalar(sf, csu0)) == 0
+if VERSION >= v"1.7"
+    @test (@ballocated benchmark_scalar(sf, csu0)) == 0
+end
 
 # TrustRegion
 function benchmark_scalar(f, u0)

From ab567dd5ca1899edfde1bb66c57142cf77b9d104 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 16 Jan 2023 22:10:22 -0500
Subject: [PATCH 047/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 3095dabff..6444085b3 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.5"
+version = "0.1.6"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"

From c8daa1fc30f2ab77e3d60ea8d633455a2b48d09d Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Jan 2023 08:31:50 -0500
Subject: [PATCH 048/375] TrustRegion -> SimpleTrustRegion and specialize the
 number types

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 27 +++++++++++----------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 289987b31..2bfb633f1 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -1,6 +1,6 @@
 """
 ```julia
-TrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
+SimpleTrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
                                autodiff = Val{true}(), diff_type = Val{:forward})
 ```
 
@@ -49,16 +49,16 @@ solver
 - `max_shrink_times`: the maximum number of times to shrink the trust region radius in a
   row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`.
 """
-struct TrustRegion{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
-    max_trust_radius::Number
-    initial_trust_radius::Number
-    step_threshold::Number
-    shrink_threshold::Number
-    expand_threshold::Number
-    shrink_factor::Number
-    expand_factor::Number
+struct SimpleTrustRegion{T, CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
+    max_trust_radius::T
+    initial_trust_radius::T
+    step_threshold::T
+    shrink_threshold::T
+    expand_threshold::T
+    shrink_factor::T
+    expand_factor::T
     max_shrink_times::Int
-    function TrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
+    function SimpleTrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
                          autodiff = Val{true}(),
                          diff_type = Val{:forward},
                          initial_trust_radius::Number = max_trust_radius / 11,
@@ -68,8 +68,9 @@ struct TrustRegion{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
                          shrink_factor::Number = 0.25,
                          expand_factor::Number = 2.0,
                          max_shrink_times::Int = 32)
-        new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
-            SciMLBase._unwrap_val(diff_type)}(max_trust_radius, initial_trust_radius,
+        new{typeof(initial_trust_radius), SciMLBase._unwrap_val(chunk_size), 
+            SciMLBase._unwrap_val(autodiff), SciMLBase._unwrap_val(diff_type)}(
+                                              max_trust_radius, initial_trust_radius,
                                               step_threshold,
                                               shrink_threshold, expand_threshold,
                                               shrink_factor,
@@ -78,7 +79,7 @@ struct TrustRegion{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-                           alg::TrustRegion, args...; abstol = nothing,
+                           alg::SimpleTrustRegion, args...; abstol = nothing,
                            reltol = nothing,
                            maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)

From b313722e4671363430ec5a39cbabd9b8476fcdd1 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Jan 2023 08:37:50 -0500
Subject: [PATCH 049/375] Complete the name change

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 28 ++++++++++-----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index cd960b664..bbee5ff09 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -52,10 +52,10 @@ if VERSION >= v"1.7"
     @test (@ballocated benchmark_scalar(sf, csu0)) == 0
 end
 
-# TrustRegion
+# SimpleTrustRegion
 function benchmark_scalar(f, u0)
     probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, TrustRegion(10.0)))
+    sol = (solve(probN, SimpleTrustRegion(10.0)))
 end
 
 sol = benchmark_scalar(sf, csu0)
@@ -69,7 +69,7 @@ using ForwardDiff
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
 for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-    TrustRegion(10.0)]
+    SimpleTrustRegion(10.0)]
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, abstol = 1e-9)
@@ -85,7 +85,7 @@ end
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
 for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-    TrustRegion(10.0)]
+    SimpleTrustRegion(10.0)]
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -127,7 +127,7 @@ for alg in [Bisection(), Falsi()]
 end
 
 for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-    TrustRegion(10.0)]
+    SimpleTrustRegion(10.0)]
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -144,8 +144,8 @@ probN = NonlinearProblem(f, u0)
 
 @test solve(probN, SimpleNewtonRaphson()).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
-@test solve(probN, TrustRegion(10.0)).u[end] ≈ sqrt(2.0)
-@test solve(probN, TrustRegion(10.0; autodiff = false)).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleTrustRegion(10.0)).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleTrustRegion(10.0; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
 @test solve(probN, Klement()).u[end] ≈ sqrt(2.0)
 
@@ -159,9 +159,9 @@ for u0 in [1.0, [1, 1.0]]
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol
 
-    @test solve(probN, TrustRegion(10.0)).u ≈ sol
-    @test solve(probN, TrustRegion(10.0)).u ≈ sol
-    @test solve(probN, TrustRegion(10.0; autodiff = false)).u ≈ sol
+    @test solve(probN, SimpleTrustRegion(10.0)).u ≈ sol
+    @test solve(probN, SimpleTrustRegion(10.0)).u ≈ sol
+    @test solve(probN, SimpleTrustRegion(10.0; autodiff = false)).u ≈ sol
 
     @test solve(probN, Broyden()).u ≈ sol
 
@@ -205,7 +205,7 @@ sol = solve(probB, Bisection(; exact_left = true, exact_right = true); immutable
 @test f(sol.right, nothing) >= 0.0
 @test f(prevfloat(sol.right), nothing) <= 0.0
 
-# Test that `TrustRegion` passes a test that `SimpleNewtonRaphson` fails on.
+# Test that `SimpleTrustRegion` passes a test that `SimpleNewtonRaphson` fails on.
 u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
 global g, f
 f = (u, p) -> 0.010000000000000002 .+
@@ -219,7 +219,7 @@ f = (u, p) -> 0.010000000000000002 .+
 .-p
 g = function (p)
     probN = NonlinearProblem{false}(f, u0, p)
-    sol = solve(probN, TrustRegion(100.0))
+    sol = solve(probN, SimpleTrustRegion(100.0))
     return sol.u
 end
 p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
@@ -227,7 +227,7 @@ u = g(p)
 f(u, p)
 @test all(f(u, p) .< 1e-10)
 
-# Test kwars in `TrustRegion`
+# Test kwars in `SimpleTrustRegion`
 max_trust_radius = [10.0, 100.0, 1000.0]
 initial_trust_radius = [10.0, 1.0, 0.1]
 step_threshold = [0.0, 0.01, 0.25]
@@ -242,7 +242,7 @@ list_of_options = zip(max_trust_radius, initial_trust_radius, step_threshold,
                       expand_factor, max_shrink_times)
 for options in list_of_options
     local probN, sol, alg
-    alg = TrustRegion(options[1];
+    alg = SimpleTrustRegion(options[1];
                       initial_trust_radius = options[2],
                       step_threshold = options[3],
                       shrink_threshold = options[4],

From c251bfdafeba7f09d395938e19da93ced94800b6 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Jan 2023 08:40:36 -0500
Subject: [PATCH 050/375] format

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 34 +++++++++++----------
 lib/SimpleNonlinearSolve/test/basictests.jl | 14 ++++-----
 2 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 2bfb633f1..4d0fcc09e 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -59,22 +59,24 @@ struct SimpleTrustRegion{T, CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     expand_factor::T
     max_shrink_times::Int
     function SimpleTrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
-                         autodiff = Val{true}(),
-                         diff_type = Val{:forward},
-                         initial_trust_radius::Number = max_trust_radius / 11,
-                         step_threshold::Number = 0.1,
-                         shrink_threshold::Number = 0.25,
-                         expand_threshold::Number = 0.75,
-                         shrink_factor::Number = 0.25,
-                         expand_factor::Number = 2.0,
-                         max_shrink_times::Int = 32)
-        new{typeof(initial_trust_radius), SciMLBase._unwrap_val(chunk_size), 
-            SciMLBase._unwrap_val(autodiff), SciMLBase._unwrap_val(diff_type)}(
-                                              max_trust_radius, initial_trust_radius,
-                                              step_threshold,
-                                              shrink_threshold, expand_threshold,
-                                              shrink_factor,
-                                              expand_factor, max_shrink_times)
+                               autodiff = Val{true}(),
+                               diff_type = Val{:forward},
+                               initial_trust_radius::Number = max_trust_radius / 11,
+                               step_threshold::Number = 0.1,
+                               shrink_threshold::Number = 0.25,
+                               expand_threshold::Number = 0.75,
+                               shrink_factor::Number = 0.25,
+                               expand_factor::Number = 2.0,
+                               max_shrink_times::Int = 32)
+        new{typeof(initial_trust_radius), SciMLBase._unwrap_val(chunk_size),
+            SciMLBase._unwrap_val(autodiff), SciMLBase._unwrap_val(diff_type)}(max_trust_radius,
+                                                                               initial_trust_radius,
+                                                                               step_threshold,
+                                                                               shrink_threshold,
+                                                                               expand_threshold,
+                                                                               shrink_factor,
+                                                                               expand_factor,
+                                                                               max_shrink_times)
     end
 end
 
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index bbee5ff09..39a4f3888 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -243,13 +243,13 @@ list_of_options = zip(max_trust_radius, initial_trust_radius, step_threshold,
 for options in list_of_options
     local probN, sol, alg
     alg = SimpleTrustRegion(options[1];
-                      initial_trust_radius = options[2],
-                      step_threshold = options[3],
-                      shrink_threshold = options[4],
-                      expand_threshold = options[5],
-                      shrink_factor = options[6],
-                      expand_factor = options[7],
-                      max_shrink_times = options[8])
+                            initial_trust_radius = options[2],
+                            step_threshold = options[3],
+                            shrink_threshold = options[4],
+                            expand_threshold = options[5],
+                            shrink_factor = options[6],
+                            expand_factor = options[7],
+                            max_shrink_times = options[8])
 
     probN = NonlinearProblem(f, u0, p)
     sol = solve(probN, alg)

From 52fd1e8d1daf2ca65d267692b5d7d3393cbec382 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Jan 2023 09:02:16 -0500
Subject: [PATCH 051/375] last few TrustRegions

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 4 ++--
 lib/SimpleNonlinearSolve/src/trustRegion.jl          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index dbe856365..c077d833d 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -32,7 +32,7 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
         solve(prob_no_brack, alg(), abstol = T(1e-2))
     end
 
-    for alg in (TrustRegion(10.0),)
+    for alg in (SimpleTrustRegion(10.0),)
         solve(prob_no_brack, alg, abstol = T(1e-2))
     end
 
@@ -53,6 +53,6 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
 end end
 
 # DiffEq styled algorithms
-export Bisection, Broyden, Falsi, Klement, SimpleNewtonRaphson, TrustRegion
+export Bisection, Broyden, Falsi, Klement, SimpleNewtonRaphson, SimpleTrustRegion
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 4d0fcc09e..08d5d5129 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -97,7 +97,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
     max_shrink_times = alg.max_shrink_times
 
     if SciMLBase.isinplace(prob)
-        error("TrustRegion currently only supports out-of-place nonlinear problems")
+        error("SimpleTrustRegion currently only supports out-of-place nonlinear problems")
     end
 
     atol = abstol !== nothing ? abstol :

From c9205260c4806bc3e9455c02edb86711462fc2e7 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Jan 2023 09:54:36 -0500
Subject: [PATCH 052/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 6444085b3..696240832 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.6"
+version = "0.1.7"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"

From 4809a8c0e467865a7476074bb0c9bf241287be61 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Wed, 18 Jan 2023 20:57:20 +0100
Subject: [PATCH 053/375] Automatic choice for maximum trust region radius

---
 .../src/SimpleNonlinearSolve.jl               |  8 +--
 lib/SimpleNonlinearSolve/src/ad.jl            |  6 +-
 lib/SimpleNonlinearSolve/src/trustRegion.jl   | 62 ++++++++++++-------
 lib/SimpleNonlinearSolve/test/basictests.jl   | 32 +++++-----
 4 files changed, 60 insertions(+), 48 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index c077d833d..622595e1e 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -19,23 +19,19 @@ include("utils.jl")
 include("bisection.jl")
 include("falsi.jl")
 include("raphson.jl")
-include("ad.jl")
 include("broyden.jl")
 include("klement.jl")
 include("trustRegion.jl")
+include("ad.jl")
 
 import SnoopPrecompile
 
 SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-    for alg in (SimpleNewtonRaphson, Broyden, Klement)
+    for alg in (SimpleNewtonRaphson, Broyden, Klement, SimpleTrustRegion)
         solve(prob_no_brack, alg(), abstol = T(1e-2))
     end
 
-    for alg in (SimpleTrustRegion(10.0),)
-        solve(prob_no_brack, alg, abstol = T(1e-2))
-    end
-
     #=
     for alg in (SimpleNewtonRaphson,)
         for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 3b8d3fee0..58a6181ea 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -30,7 +30,8 @@ end
 
 function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
                                                 iip,
-                                                <:Dual{T, V, P}}, alg::SimpleNewtonRaphson,
+                                                <:Dual{T, V, P}},
+                         alg::Union{SimpleNewtonRaphson, SimpleTrustRegion},
                          args...; kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
@@ -39,7 +40,8 @@ end
 function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
                                                 iip,
                                                 <:AbstractArray{<:Dual{T, V, P}}},
-                         alg::SimpleNewtonRaphson, args...; kwargs...) where {iip, T, V, P}
+                         alg::Union{SimpleNewtonRaphson, SimpleTrustRegion}, args...;
+                         kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
                                     retcode = sol.retcode)
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 08d5d5129..329eb8808 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -1,17 +1,22 @@
 """
 ```julia
-SimpleTrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
-                               autodiff = Val{true}(), diff_type = Val{:forward})
+SimpleTrustRegion(; chunk_size = Val{0}(),
+                    autodiff = Val{true}(),
+                    diff_type = Val{:forward},
+                    max_trust_radius::Real = 0.0,
+                    initial_trust_radius::Real = 0.0,
+                    step_threshold::Real = 0.1,
+                    shrink_threshold::Real = 0.25,
+                    expand_threshold::Real = 0.75,
+                    shrink_factor::Real = 0.25,
+                    expand_factor::Real = 2.0,
+                    max_shrink_times::Int = 32
 ```
 
 A low-overhead implementation of a
 [trust-region](https://optimization.mccormick.northwestern.edu/index.php/Trust-region_methods)
 solver
 
-### Arguments
-- `max_trust_radius`: the maximum radius of the trust region. The step size in the algorithm
-  will change dynamically. However, it will never be greater than the `max_trust_radius`.
-
 ### Keyword Arguments
 
 - `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
@@ -26,6 +31,8 @@ solver
 - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
   `Val{:forward}` for forward finite differences. For more details on the choices, see the
   [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+- `max_trust_radius`: the maximum radius of the trust region. Defaults to
+  `max(norm(f(u0)), maximum(u0) - minimum(u0))`.
 - `initial_trust_radius`: the initial trust region radius. Defaults to
   `max_trust_radius / 11`.
 - `step_threshold`: the threshold for taking a step. In every iteration, the threshold is
@@ -58,25 +65,28 @@ struct SimpleTrustRegion{T, CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     shrink_factor::T
     expand_factor::T
     max_shrink_times::Int
-    function SimpleTrustRegion(max_trust_radius::Number; chunk_size = Val{0}(),
+    function SimpleTrustRegion(; chunk_size = Val{0}(),
                                autodiff = Val{true}(),
                                diff_type = Val{:forward},
-                               initial_trust_radius::Number = max_trust_radius / 11,
-                               step_threshold::Number = 0.1,
-                               shrink_threshold::Number = 0.25,
-                               expand_threshold::Number = 0.75,
-                               shrink_factor::Number = 0.25,
-                               expand_factor::Number = 2.0,
+                               max_trust_radius::Real = 0.0,
+                               initial_trust_radius::Real = 0.0,
+                               step_threshold::Real = 0.1,
+                               shrink_threshold::Real = 0.25,
+                               expand_threshold::Real = 0.75,
+                               shrink_factor::Real = 0.25,
+                               expand_factor::Real = 2.0,
                                max_shrink_times::Int = 32)
-        new{typeof(initial_trust_radius), SciMLBase._unwrap_val(chunk_size),
-            SciMLBase._unwrap_val(autodiff), SciMLBase._unwrap_val(diff_type)}(max_trust_radius,
-                                                                               initial_trust_radius,
-                                                                               step_threshold,
-                                                                               shrink_threshold,
-                                                                               expand_threshold,
-                                                                               shrink_factor,
-                                                                               expand_factor,
-                                                                               max_shrink_times)
+        new{typeof(initial_trust_radius),
+            SciMLBase._unwrap_val(chunk_size),
+            SciMLBase._unwrap_val(autodiff),
+            SciMLBase._unwrap_val(diff_type)}(max_trust_radius,
+                                              initial_trust_radius,
+                                              step_threshold,
+                                              shrink_threshold,
+                                              expand_threshold,
+                                              shrink_factor,
+                                              expand_factor,
+                                              max_shrink_times)
     end
 end
 
@@ -114,6 +124,14 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         ∇f = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x), F)
     end
 
+    # Set default trust region radius if not specified by user.
+    if Δₘₐₓ == 0.0
+        Δₘₐₓ = max(norm(F), maximum(x) - minimum(x))
+    end
+    if Δ == 0.0
+        Δ = Δₘₐₓ / 11
+    end
+
     fₖ = 0.5 * norm(F)^2
     H = ∇f * ∇f
     g = ∇f * F
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 39a4f3888..98687bc07 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -55,7 +55,7 @@ end
 # SimpleTrustRegion
 function benchmark_scalar(f, u0)
     probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, SimpleTrustRegion(10.0)))
+    sol = (solve(probN, SimpleTrustRegion()))
 end
 
 sol = benchmark_scalar(sf, csu0)
@@ -68,8 +68,7 @@ using ForwardDiff
 # Immutable
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
-for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-    SimpleTrustRegion(10.0)]
+for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion())
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, abstol = 1e-9)
@@ -84,8 +83,7 @@ end
 
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
-for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-    SimpleTrustRegion(10.0)]
+for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion())
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -126,8 +124,7 @@ for alg in [Bisection(), Falsi()]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
 
-for alg in [SimpleNewtonRaphson(), Broyden(), Klement(),
-    SimpleTrustRegion(10.0)]
+for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion())
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -144,8 +141,8 @@ probN = NonlinearProblem(f, u0)
 
 @test solve(probN, SimpleNewtonRaphson()).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleTrustRegion(10.0)).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleTrustRegion(10.0; autodiff = false)).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleTrustRegion()).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleTrustRegion(; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
 @test solve(probN, Klement()).u[end] ≈ sqrt(2.0)
 
@@ -159,9 +156,9 @@ for u0 in [1.0, [1, 1.0]]
     @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
     @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol
 
-    @test solve(probN, SimpleTrustRegion(10.0)).u ≈ sol
-    @test solve(probN, SimpleTrustRegion(10.0)).u ≈ sol
-    @test solve(probN, SimpleTrustRegion(10.0; autodiff = false)).u ≈ sol
+    @test solve(probN, SimpleTrustRegion()).u ≈ sol
+    @test solve(probN, SimpleTrustRegion()).u ≈ sol
+    @test solve(probN, SimpleTrustRegion(; autodiff = false)).u ≈ sol
 
     @test solve(probN, Broyden()).u ≈ sol
 
@@ -215,17 +212,16 @@ f = (u, p) -> 0.010000000000000002 .+
                  (0.21640425613334457 .+
                   216.40425613334457 ./
                   (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
-              0.0011552453009332421u
-.-p
+              0.0011552453009332421u .- p
 g = function (p)
     probN = NonlinearProblem{false}(f, u0, p)
-    sol = solve(probN, SimpleTrustRegion(100.0))
+    sol = solve(probN, SimpleTrustRegion())
     return sol.u
 end
 p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
 u = g(p)
 f(u, p)
-@test all(f(u, p) .< 1e-10)
+@test all(abs.(f(u, p)) .< 1e-10)
 
 # Test kwars in `SimpleTrustRegion`
 max_trust_radius = [10.0, 100.0, 1000.0]
@@ -242,7 +238,7 @@ list_of_options = zip(max_trust_radius, initial_trust_radius, step_threshold,
                       expand_factor, max_shrink_times)
 for options in list_of_options
     local probN, sol, alg
-    alg = SimpleTrustRegion(options[1];
+    alg = SimpleTrustRegion(max_trust_radius = options[1],
                             initial_trust_radius = options[2],
                             step_threshold = options[3],
                             shrink_threshold = options[4],
@@ -253,5 +249,5 @@ for options in list_of_options
 
     probN = NonlinearProblem(f, u0, p)
     sol = solve(probN, alg)
-    @test all(f(u, p) .< 1e-10)
+    @test all(abs.(f(u, p)) .< 1e-10)
 end

From 84e2dd1cf58dfd7b2331b72434b3072059a83862 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 18 Jan 2023 15:19:19 -0500
Subject: [PATCH 054/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 696240832..a12faeac9 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.7"
+version = "0.1.8"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"

From ea3a2bfc91a50730b652541c5b3d5ee83e10a458 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Sun, 29 Jan 2023 15:40:32 +0100
Subject: [PATCH 055/375] Implementation of Ridder

---
 .../src/SimpleNonlinearSolve.jl               |  5 +-
 lib/SimpleNonlinearSolve/src/ridder.jl        | 84 +++++++++++++++++++
 lib/SimpleNonlinearSolve/test/basictests.jl   | 18 +++-
 3 files changed, 104 insertions(+), 3 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/ridder.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 622595e1e..c29fe63d6 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -22,6 +22,7 @@ include("raphson.jl")
 include("broyden.jl")
 include("klement.jl")
 include("trustRegion.jl")
+include("ridder.jl")
 include("ad.jl")
 
 import SnoopPrecompile
@@ -43,12 +44,12 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     =#
 
     prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p, T.((0.0, 2.0)), T(2))
-    for alg in (Bisection, Falsi)
+    for alg in (Bisection, Falsi, Ridder)
         solve(prob_brack, alg(), abstol = T(1e-2))
     end
 end end
 
 # DiffEq styled algorithms
-export Bisection, Broyden, Falsi, Klement, SimpleNewtonRaphson, SimpleTrustRegion
+export Bisection, Broyden, Falsi, Klement, Ridder, SimpleNewtonRaphson, SimpleTrustRegion
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
new file mode 100644
index 000000000..bb5c8d4db
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -0,0 +1,84 @@
+"""
+`Ridder()`
+
+A non-allocating ridder method
+
+"""
+struct Ridder <: AbstractBracketingAlgorithm end
+
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
+                         maxiters = 1000,
+                         kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    left, right = prob.tspan
+    fl, fr = f(left), f(right)
+
+    if iszero(fl)
+        return SciMLBase.build_solution(prob, alg, left, fl;
+                                        retcode = ReturnCode.ExactSolutionLeft, left = left,
+                                        right = right)
+    end
+
+    xo = oftype(left, Inf)
+    i = 1
+    if !iszero(fr)
+        while i < maxiters
+            mid = (left + right) / 2
+            (mid == left || mid == right) &&
+                return SciMLBase.build_solution(prob, alg, left, fl;
+                                                retcode = ReturnCode.FloatingPointLimit,
+                                                left = left, right = right)
+            fm = f(mid)
+            s = sqrt(fm^2 - fl * fr)
+            iszero(s) &&
+                return SciMLBase.build_solution(prob, alg, left, fl;
+                                                retcode = ReturnCode.Failure,
+                                                left = left, right = right)
+            x = mid + (mid - left) * sign(fl - fr) * fm / s
+            fx = f(x)
+            xo = x
+            if iszero(fx)
+                right = x
+                fr = fx
+                break
+            end
+            if sign(fx) != sign(fm)
+                left = mid
+                fl = fm
+                right = x
+                fr = fx
+            elseif sign(fx) != sign(fl)
+                right = x
+                fr = fx
+            else
+                @assert sign(fx) != sign(fr)
+                left = x
+                fl = fx
+            end
+            i += 1
+        end
+    end
+
+    while i < maxiters
+        mid = (left + right) / 2
+        (mid == left || mid == right) &&
+            return SciMLBase.build_solution(prob, alg, left, fl;
+                                            retcode = ReturnCode.FloatingPointLimit,
+                                            left = left, right = right)
+        fm = f(mid)
+        if iszero(fm)
+            right = mid
+            fr = fm
+        elseif sign(fm) == sign(fl)
+            left = mid
+            fl = fm
+        else
+            right = mid
+            fr = fm
+        end
+        i += 1
+    end
+
+    return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
+                                    left = left, right = right)
+end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 98687bc07..d68b3f42c 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -109,10 +109,22 @@ for p in 1.1:0.1:100.0
     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
 end
 
+# Ridder
+g = function (p)
+    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
+    sol = solve(probN, Ridder())
+    return sol.left
+end
+
+for p in 1.1:0.1:100.0
+    @test g(p) ≈ sqrt(p)
+    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+end
+
 f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
 t = (p) -> [sqrt(p[2] / p[1])]
 p = [0.9, 50.0]
-for alg in [Bisection(), Falsi()]
+for alg in [Bisection(), Falsi(), Ridder()]
     global g, p
     g = function (p)
         probN = IntervalNonlinearProblem{false}(f, tspan, p)
@@ -173,6 +185,10 @@ probB = IntervalNonlinearProblem(f, tspan)
 sol = solve(probB, Falsi())
 @test sol.left ≈ sqrt(2.0)
 
+# Ridder
+sol = solve(probB, Ridder())
+@test sol.left ≈ sqrt(2.0)
+
 sol = solve(probB, Bisection())
 @test sol.left ≈ sqrt(2.0)
 

From e84be4680dadadc1fcca5bf4bd37b6742934f348 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Sun, 29 Jan 2023 15:57:54 +0100
Subject: [PATCH 056/375] Small change to Ridder

---
 lib/SimpleNonlinearSolve/src/ridder.jl | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index bb5c8d4db..9fe7bccfe 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -69,12 +69,9 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
         if iszero(fm)
             right = mid
             fr = fm
-        elseif sign(fm) == sign(fl)
+        else
             left = mid
             fl = fm
-        else
-            right = mid
-            fr = fm
         end
         i += 1
     end

From fdfb03b7ccab66b2bf683997df97b8f20a9480a7 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Sun, 29 Jan 2023 17:19:35 +0100
Subject: [PATCH 057/375] Added more tests

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index d68b3f42c..4ee2c3ab7 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -185,12 +185,21 @@ probB = IntervalNonlinearProblem(f, tspan)
 sol = solve(probB, Falsi())
 @test sol.left ≈ sqrt(2.0)
 
+sol = solve(probB, Bisection())
+@test sol.left ≈ sqrt(2.0)
+
 # Ridder
 sol = solve(probB, Ridder())
 @test sol.left ≈ sqrt(2.0)
-
-sol = solve(probB, Bisection())
+tspan = (sqrt(2.0), 10.0)
+probB = IntervalNonlinearProblem(f, tspan)
+sol = solve(probB, Ridder())
 @test sol.left ≈ sqrt(2.0)
+tspan = (0.0, sqrt(2.0))
+probB = IntervalNonlinearProblem(f, tspan)
+sol = solve(probB, Ridder())
+@test sol.left ≈ sqrt(2.0)
+probB = IntervalNonlinearProblem(f, tspan)
 
 # Garuntee Tests for Bisection
 f = function (u, p)

From 2959eda04792582ef18379016af3a2703ae94415 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Sun, 29 Jan 2023 17:21:33 +0100
Subject: [PATCH 058/375] Small fix in the tests

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 4ee2c3ab7..017569227 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -199,7 +199,6 @@ tspan = (0.0, sqrt(2.0))
 probB = IntervalNonlinearProblem(f, tspan)
 sol = solve(probB, Ridder())
 @test sol.left ≈ sqrt(2.0)
-probB = IntervalNonlinearProblem(f, tspan)
 
 # Garuntee Tests for Bisection
 f = function (u, p)

From 8e57137551ad9b7360d9294fcfc84a287d80f042 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Mon, 30 Jan 2023 13:02:03 +0100
Subject: [PATCH 059/375] Adding a Brent method

---
 .../src/SimpleNonlinearSolve.jl               |   6 +-
 lib/SimpleNonlinearSolve/src/brent.jl         | 109 ++++++++++++++++++
 lib/SimpleNonlinearSolve/test/basictests.jl   |  26 ++++-
 3 files changed, 138 insertions(+), 3 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/brent.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index c29fe63d6..33f5895d7 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -23,6 +23,7 @@ include("broyden.jl")
 include("klement.jl")
 include("trustRegion.jl")
 include("ridder.jl")
+include("brent.jl")
 include("ad.jl")
 
 import SnoopPrecompile
@@ -44,12 +45,13 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     =#
 
     prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p, T.((0.0, 2.0)), T(2))
-    for alg in (Bisection, Falsi, Ridder)
+    for alg in (Bisection, Falsi, Ridder, Brent)
         solve(prob_brack, alg(), abstol = T(1e-2))
     end
 end end
 
 # DiffEq styled algorithms
-export Bisection, Broyden, Falsi, Klement, Ridder, SimpleNewtonRaphson, SimpleTrustRegion
+export Bisection, Brent, Broyden, Falsi, Klement, Ridder, SimpleNewtonRaphson,
+       SimpleTrustRegion
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
new file mode 100644
index 000000000..9a60abfee
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -0,0 +1,109 @@
+"""
+`Brent()`
+
+A non-allocating Brent method
+
+"""
+struct Brent <: AbstractBracketingAlgorithm end
+
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
+                         maxiters = 1000,
+                         kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    a, b = prob.tspan
+    fa, fb = f(a), f(b)
+
+    if iszero(fa)
+        return SciMLBase.build_solution(prob, alg, a, fa;
+                                        retcode = ReturnCode.ExactSolutionLeft, left = a,
+                                        right = b)
+    end
+    if abs(fa) < abs(fb)
+        c = b
+        b = a
+        a = c
+        tmp = fa
+        fa = fb
+        fb = tmp
+    end
+
+    c = a
+    d = c
+    i = 1
+    cond = true
+    if !iszero(fb)
+        while i < maxiters
+            fc = f(c)
+            if fa != fc && fb != fc
+                # Inverse quadratic interpolation
+                s = a * fb * fc / ((fa - fb) * (fa - fc)) +
+                    b * fa * fc / ((fb - fa) * (fb - fc)) +
+                    c * fa * fb / ((fc - fa) * (fc - fb))
+            else
+                # Secant method
+                s = b - fb * (b - a) / (fb - fa)
+            end
+            if (s < min((3 * a + b) / 4, b) || s > max((3 * a + b) / 4, b)) ||
+               (cond && abs(s - b) ≥ abs(b - c) / 2) ||
+               (!cond && abs(s - b) ≥ abs(c - d) / 2) ||
+               (cond && abs(b - c) ≤ eps(a)) ||
+               (!cond && abs(c - d) ≤ eps(a))
+                # Bisection method
+                s = (a + b) / 2
+                (s == a || s == b) &&
+                    return SciMLBase.build_solution(prob, alg, a, fa;
+                                                    retcode = ReturnCode.FloatingPointLimit,
+                                                    left = a, right = b)
+                cond = true
+            else
+                cond = false
+            end
+            fs = f(s)
+            if iszero(fs)
+                a = b
+                b = s
+                break
+            end
+            if fa * fs < 0
+                d = c
+                c = b
+                b = s
+                fb = fs
+            else
+                a = s
+                fa = fs
+            end
+            if abs(fa) < abs(fb)
+                d = c
+                c = b
+                b = a
+                a = c
+                fc = fb
+                fb = fa
+                fa = fc
+            end
+            i += 1
+        end
+    end
+
+    while i < maxiters
+        c = (a + b) / 2
+        if (c == a || c == b)
+            return SciMLBase.build_solution(prob, alg, a, fa;
+                                            retcode = ReturnCode.FloatingPointLimit,
+                                            left = a, right = b)
+        end
+        fc = f(c)
+        if iszero(fc)
+            b = c
+            fb = fc
+        else
+            a = c
+            fa = fc
+        end
+        i += 1
+    end
+    
+    return SciMLBase.build_solution(prob, alg, a, fa; retcode = ReturnCode.MaxIters,
+                                    left = a, right = b)
+end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 017569227..fa15e798e 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -121,10 +121,22 @@ for p in 1.1:0.1:100.0
     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
 end
 
+# Brent
+g = function (p)
+    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
+    sol = solve(probN, Brent())
+    return sol.left
+end
+
+for p in 1.1:0.1:100.0
+    @test g(p) ≈ sqrt(p)
+    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+end
+
 f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
 t = (p) -> [sqrt(p[2] / p[1])]
 p = [0.9, 50.0]
-for alg in [Bisection(), Falsi(), Ridder()]
+for alg in [Bisection(), Falsi(), Ridder(), Brent()]
     global g, p
     g = function (p)
         probN = IntervalNonlinearProblem{false}(f, tspan, p)
@@ -200,6 +212,18 @@ probB = IntervalNonlinearProblem(f, tspan)
 sol = solve(probB, Ridder())
 @test sol.left ≈ sqrt(2.0)
 
+# Brent
+sol = solve(probB, Brent())
+@test sol.left ≈ sqrt(2.0)
+tspan = (sqrt(2.0), 10.0)
+probB = IntervalNonlinearProblem(f, tspan)
+sol = solve(probB, Brent())
+@test sol.left ≈ sqrt(2.0)
+tspan = (0.0, sqrt(2.0))
+probB = IntervalNonlinearProblem(f, tspan)
+sol = solve(probB, Brent())
+@test sol.left ≈ sqrt(2.0)
+
 # Garuntee Tests for Bisection
 f = function (u, p)
     if u < 2.0

From c9ae3799c6134b5ec2dff75f2459f75b2a67f87a Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Mon, 30 Jan 2023 16:21:00 +0100
Subject: [PATCH 060/375] Updating Brent

---
 lib/SimpleNonlinearSolve/src/brent.jl | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index 9a60abfee..99f645f6a 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -12,6 +12,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     fa, fb = f(a), f(b)
+    ϵ = eps(convert(typeof(fa), 1.0))
 
     if iszero(fa)
         return SciMLBase.build_solution(prob, alg, a, fa;
@@ -46,8 +47,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
             if (s < min((3 * a + b) / 4, b) || s > max((3 * a + b) / 4, b)) ||
                (cond && abs(s - b) ≥ abs(b - c) / 2) ||
                (!cond && abs(s - b) ≥ abs(c - d) / 2) ||
-               (cond && abs(b - c) ≤ eps(a)) ||
-               (!cond && abs(c - d) ≤ eps(a))
+               (cond && abs(b - c) ≤ ϵ) ||
+               (!cond && abs(c - d) ≤ ϵ)
                 # Bisection method
                 s = (a + b) / 2
                 (s == a || s == b) &&
@@ -60,8 +61,12 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
             end
             fs = f(s)
             if iszero(fs)
-                a = b
+                if b < a
+                    a = b
+                    fa = fb
+                end
                 b = s
+                fb = fs
                 break
             end
             if fa * fs < 0
@@ -103,7 +108,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
         end
         i += 1
     end
-    
+
     return SciMLBase.build_solution(prob, alg, a, fa; retcode = ReturnCode.MaxIters,
                                     left = a, right = b)
 end

From b7eb5620f4074f277fd9ac25cf61aac3b6f2fcb0 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 31 Jan 2023 06:03:21 -0500
Subject: [PATCH 061/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index a12faeac9..3b8b5d859 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.8"
+version = "0.1.9"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"

From 2f6e0838c80b6d433e982803dc9623a7283cd494 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Mon, 6 Feb 2023 13:03:47 +0100
Subject: [PATCH 062/375] Started to implement DFSane

---
 .../src/SimpleNonlinearSolve.jl               |   5 +-
 lib/SimpleNonlinearSolve/src/ad.jl            |   6 +-
 lib/SimpleNonlinearSolve/src/dfsane.jl        | 135 ++++++++++++++++++
 lib/SimpleNonlinearSolve/test/basictests.jl   |  72 ++++++++--
 4 files changed, 205 insertions(+), 13 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/dfsane.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 33f5895d7..940db0835 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -24,13 +24,14 @@ include("klement.jl")
 include("trustRegion.jl")
 include("ridder.jl")
 include("brent.jl")
+include("dfsane.jl")
 include("ad.jl")
 
 import SnoopPrecompile
 
 SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-    for alg in (SimpleNewtonRaphson, Broyden, Klement, SimpleTrustRegion)
+    for alg in (SimpleNewtonRaphson, Broyden, Klement, SimpleTrustRegion, SimpleDFSane)
         solve(prob_no_brack, alg(), abstol = T(1e-2))
     end
 
@@ -51,7 +52,7 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
 end end
 
 # DiffEq styled algorithms
-export Bisection, Brent, Broyden, Falsi, Klement, Ridder, SimpleNewtonRaphson,
+export Bisection, Brent, Broyden, SimpleDFSane, Falsi, Klement, Ridder, SimpleNewtonRaphson,
        SimpleTrustRegion
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 58a6181ea..47713439c 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -31,16 +31,16 @@ end
 function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
                                                 iip,
                                                 <:Dual{T, V, P}},
-                         alg::Union{SimpleNewtonRaphson, SimpleTrustRegion},
+                         alg::Union{SimpleNewtonRaphson, SimpleTrustRegion, SimpleDFSane}, # TODO make this to one variable
                          args...; kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
                                     retcode = sol.retcode)
 end
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector}, # TODO make this to one variable
                                                 iip,
                                                 <:AbstractArray{<:Dual{T, V, P}}},
-                         alg::Union{SimpleNewtonRaphson, SimpleTrustRegion}, args...;
+                         alg::Union{SimpleNewtonRaphson, SimpleTrustRegion, SimpleDFSane}, args...;
                          kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
new file mode 100644
index 000000000..a66703dab
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -0,0 +1,135 @@
+"""
+```julia
+SimpleDFSane(; σ_min::Real = 1e-10, σ_0::Real = 1.0, M::Int = 10,
+            γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+            nexp::Int = 2)
+```
+
+A low-overhead implementation of the df-sane method. For more information, see [1].
+References:
+    W LaCruz, JM Martinez, and M Raydan (2006), Spectral residual mathod without gradient information for solving large-scale nonlinear systems of equations, Mathematics of Computation, 75, 1429-1448.
+### Keyword Arguments
+
+
+- `σ_min`: the minimum value of `σ_k`. Defaults to `1e-10`. # TODO write about this...
+- `σ_0`: the initial value of `σ_k`. Defaults to `1.0`. # TODO write about this...
+- `M`: the value of `M` in the paper. Defaults to `10`. # TODO write about this...
+- `γ`: the value of `γ` in the paper. Defaults to `1e-4`. # TODO write about this...
+- `τ_min`: the minimum value of `τ_k`. Defaults to `0.1`. # TODO write about this...
+- `τ_max`: the maximum value of `τ_k`. Defaults to `0.5`. # TODO write about this...
+- `nexp`: the value of `nexp` in the paper. Defaults to `2`. # TODO write about this...
+
+"""
+struct SimpleDFSane{T} <: AbstractSimpleNonlinearSolveAlgorithm
+    σ_min::T
+    σ_0::T
+    M::Int
+    γ::T
+    τ_min::T
+    τ_max::T
+    nexp::Int
+
+    function SimpleDFSane(; σ_min::Real = 1e-10, σ_0::Real = 1.0, M::Int = 10,
+                          γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+                          nexp::Int = 2)
+        new{typeof(σ_min)}(σ_min, σ_0, M, γ, τ_min, τ_max, nexp)
+    end
+end
+
+function SciMLBase.__solve(prob::NonlinearProblem,
+                           alg::SimpleDFSane, args...; abstol = nothing,
+                           reltol = nothing,
+                           maxiters = 1000, kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    x = float(prob.u0)
+    T = eltype(x)
+    σ_min = float(alg.σ_min)
+    σ_k = float(alg.σ_0)
+    M = alg.M
+    γ = float(alg.γ)
+    τ_min = float(alg.τ_min)
+    τ_max = float(alg.τ_max)
+    nexp = alg.nexp
+
+    if SciMLBase.isinplace(prob)
+        error("SimpleDFSane currently only supports out-of-place nonlinear problems")
+    end
+
+    atol = abstol !== nothing ? abstol :
+           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
+    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+
+    function ff(x)
+        F = f(x)
+        f_k = norm(F)^nexp
+        return f_k, F
+    end
+
+    f_k, F_k = ff(x)
+    α_0 = convert(T, 1.0)
+    f_0 = f_k
+    prev_fs = fill(f_k, M)
+
+    for k in 1:maxiters
+        iszero(F_k) &&
+            return SciMLBase.build_solution(prob, alg, x, F_k;
+                                            retcode = ReturnCode.Success)
+
+        # Control spectral parameter
+        if abs(σ_k) > 1 / σ_min
+            σ_k = 1 / σ_min * sign(σ_k)
+        elseif abs(σ_k) < σ_min
+            σ_k = σ_min
+        end
+
+        # Line search direction
+        d = -σ_k * F_k
+
+        # Nonmonotone line search
+        η = f_0 / k^2
+
+        f_bar = maximum(prev_fs)
+        α_p = α_0
+        α_m = α_0
+        xp = x + α_p * d
+        fp, Fp = ff(xp)
+        while true
+            if fp ≤ f_bar + η - γ * α_p^2 * f_k
+                break
+            end
+
+            α_tp = α_p^2 * f_k / (fp + (2 * α_p - 1) * f_k)
+            xp = x - α_m * d
+            fp, Fp = ff(xp)
+
+            if fp ≤ f_bar + η - γ * α_m^2 * f_k
+                break
+            end
+
+            α_tm = α_m^2 * f_k / (fp + (2 * α_m - 1) * f_k)
+            α_p = min(τ_max * α_p, max(α_tp, τ_min * α_p))
+            α_m = min(τ_max * α_m, max(α_tm, τ_min * α_m))
+            xp = x + α_p * d
+            fp, Fp = ff(xp)
+        end
+
+        if isapprox(xp, x, atol = atol, rtol = rtol)
+            return SciMLBase.build_solution(prob, alg, xp, Fp;
+                                            retcode = ReturnCode.Success)
+        end
+        # Update spectral parameter
+        s_k = xp - x
+        y_k = Fp - F_k
+        σ_k = dot(s_k, s_k) / dot(s_k, y_k)
+
+        # Take step
+        x = xp
+        F_k = Fp
+        f_k = fp
+
+        # Store function value
+        idx_to_replace = k % M + 1
+        prev_fs[idx_to_replace] = fp
+    end
+    return SciMLBase.build_solution(prob, alg, x, F_k; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index fa15e798e..782c26625 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -62,13 +62,24 @@ sol = benchmark_scalar(sf, csu0)
 @test sol.retcode === ReturnCode.Success
 @test sol.u * sol.u - 2 < 1e-9
 
+# SimpleDFSane
+function benchmark_scalar(f, u0)
+    probN = NonlinearProblem{false}(f, u0)
+    sol = (solve(probN, SimpleDFSane()))
+end
+
+sol = benchmark_scalar(sf, csu0)
+@test sol.retcode === ReturnCode.Success
+@test sol.u * sol.u - 2 < 1e-9
+
 # AD Tests
 using ForwardDiff
 
 # Immutable
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
-for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion())
+for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion(),
+            SimpleDFSane())
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, abstol = 1e-9)
@@ -76,14 +87,15 @@ for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion())
     end
 
     for p in 1.1:0.1:100.0
-        @test g(p) ≈ sqrt(p)
-        @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+        @test abs.(g(p)) ≈ sqrt(p)
+        @test abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
     end
 end
 
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
-for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion())
+for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion(),
+            SimpleDFSane())
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -91,8 +103,8 @@ for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion())
     end
 
     for p in 1.1:0.1:100.0
-        @test g(p) ≈ sqrt(p)
-        @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+        @test abs(g(p)) ≈ sqrt(p)
+        @test abs(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
     end
 end
 
@@ -148,7 +160,7 @@ for alg in [Bisection(), Falsi(), Ridder(), Brent()]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
 
-for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion())
+for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion(), SimpleDFSane())
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -169,6 +181,7 @@ probN = NonlinearProblem(f, u0)
 @test solve(probN, SimpleTrustRegion(; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
 @test solve(probN, Klement()).u[end] ≈ sqrt(2.0)
+@test solve(probN, SimpleDFSane()).u[end] ≈ sqrt(2.0)
 
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol
@@ -185,8 +198,8 @@ for u0 in [1.0, [1, 1.0]]
     @test solve(probN, SimpleTrustRegion(; autodiff = false)).u ≈ sol
 
     @test solve(probN, Broyden()).u ≈ sol
-
     @test solve(probN, Klement()).u ≈ sol
+    @test solve(probN, SimpleDFSane()).u ≈ sol
 end
 
 # Bisection Tests
@@ -299,3 +312,46 @@ for options in list_of_options
     sol = solve(probN, alg)
     @test all(abs.(f(u, p)) .< 1e-10)
 end
+
+# # Test that `SimpleDFSane` passes a test that `SimpleNewtonRaphson` fails on.
+# u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
+# global g, f
+# f = (u, p) -> 0.010000000000000002 .+
+#               10.000000000000002 ./ (1 .+
+#                (0.21640425613334457 .+
+#                 216.40425613334457 ./ (1 .+
+#                  (0.21640425613334457 .+
+#                   216.40425613334457 ./
+#                   (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
+#               0.0011552453009332421u .- p
+# g = function (p)
+#     probN = NonlinearProblem{false}(f, u0, p)
+#     sol = solve(probN, SimpleDFSane())
+#     return sol.u
+# end
+# p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+# u = g(p)
+# f(u, p)
+# @test all(abs.(f(u, p)) .< 1e-10)
+
+# # Test kwars in `SimpleDFSane`
+
+
+# list_of_options = zip(max_trust_radius, initial_trust_radius, step_threshold,
+#                       shrink_threshold, expand_threshold, shrink_factor,
+#                       expand_factor, max_shrink_times)
+# for options in list_of_options
+#     local probN, sol, alg
+#     alg = SimpleDFSane(max_trust_radius = options[1],
+#                             initial_trust_radius = options[2],
+#                             step_threshold = options[3],
+#                             shrink_threshold = options[4],
+#                             expand_threshold = options[5],
+#                             shrink_factor = options[6],
+#                             expand_factor = options[7],
+#                             max_shrink_times = options[8])
+
+#     probN = NonlinearProblem(f, u0, p)
+#     sol = solve(probN, alg)
+#     @test all(abs.(f(u, p)) .< 1e-10)
+# end

From f816305756032dc14aa4ed6992c95ba6e96cbf2d Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Mon, 6 Feb 2023 19:51:25 +0100
Subject: [PATCH 063/375] DFSane implementation

---
 lib/SimpleNonlinearSolve/src/ad.jl          |   6 +-
 lib/SimpleNonlinearSolve/src/dfsane.jl      | 136 +++++++++++---------
 lib/SimpleNonlinearSolve/test/basictests.jl |  99 +++++++-------
 3 files changed, 137 insertions(+), 104 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 47713439c..dbd90243a 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -31,16 +31,16 @@ end
 function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
                                                 iip,
                                                 <:Dual{T, V, P}},
-                         alg::Union{SimpleNewtonRaphson, SimpleTrustRegion, SimpleDFSane}, # TODO make this to one variable
+                         alg::AbstractSimpleNonlinearSolveAlgorithm,
                          args...; kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
                                     retcode = sol.retcode)
 end
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector}, # TODO make this to one variable
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
                                                 iip,
                                                 <:AbstractArray{<:Dual{T, V, P}}},
-                         alg::Union{SimpleNewtonRaphson, SimpleTrustRegion, SimpleDFSane}, args...;
+                         alg::AbstractSimpleNonlinearSolveAlgorithm, args...;
                          kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index a66703dab..3d2e9c8aa 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -1,55 +1,78 @@
 """
 ```julia
-SimpleDFSane(; σ_min::Real = 1e-10, σ_0::Real = 1.0, M::Int = 10,
-            γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
-            nexp::Int = 2)
+SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
+             M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+             nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 / k^2)
 ```
 
-A low-overhead implementation of the df-sane method. For more information, see [1].
-References:
-    W LaCruz, JM Martinez, and M Raydan (2006), Spectral residual mathod without gradient information for solving large-scale nonlinear systems of equations, Mathematics of Computation, 75, 1429-1448.
-### Keyword Arguments
-
+A low-overhead implementation of the df-sane method for solving large-scale nonlinear
+systems of equations. For in depth information about all the parameters and the algorithm,
+see the paper: [W LaCruz, JM Martinez, and M Raydan (2006), Spectral residual mathod without
+gradient information for solving large-scale nonlinear systems of equations, Mathematics of
+Computation, 75, 1429-1448.](https://www.researchgate.net/publication/220576479_Spectral_Residual_Method_without_Gradient_Information_for_Solving_Large-Scale_Nonlinear_Systems_of_Equations)
 
-- `σ_min`: the minimum value of `σ_k`. Defaults to `1e-10`. # TODO write about this...
-- `σ_0`: the initial value of `σ_k`. Defaults to `1.0`. # TODO write about this...
-- `M`: the value of `M` in the paper. Defaults to `10`. # TODO write about this...
-- `γ`: the value of `γ` in the paper. Defaults to `1e-4`. # TODO write about this...
-- `τ_min`: the minimum value of `τ_k`. Defaults to `0.1`. # TODO write about this...
-- `τ_max`: the maximum value of `τ_k`. Defaults to `0.5`. # TODO write about this...
-- `nexp`: the value of `nexp` in the paper. Defaults to `2`. # TODO write about this...
+### Keyword Arguments
 
+- `σ_min`: the minimum value of the spectral coefficient `σ_k` which is related to the step
+  size in the algorithm. Defaults to `1e-10`.
+- `σ_max`: the maximum value of the spectral coefficient `σ_k` which is related to the step
+  size in the algorithm. Defaults to `1e10`.
+- `σ_1`: the initial value of the spectral coefficient `σ_k` which is related to the step
+  size in the algorithm.. Defaults to `1.0`.
+- `M`: The monotonicity of the algorithm is determined by a this positive integer.
+  A value of 1 for `M` would result in strict monotonicity in the decrease of the L2-norm
+  of the function `f`. However, higher values allow for more flexibility in this reduction.
+  Despite this, the algorithm still ensures global convergence through the use of a
+  non-monotone line-search algorithm that adheres to the Grippo-Lampariello-Lucidi
+  condition. Values in the range of 5 to 20 are usually sufficient, but some cases may call
+  for a higher value of `M`. The default setting is 10.
+- `γ`: a parameter that influences if a proposed step will be accepted. Higher value of `γ`
+  will make the algorithm more restrictive in accepting steps. Defaults to `1e-4`.
+- `τ_min`: if a step is rejected the new step size will get multiplied by factor, and this
+  parameter is the minimum value of that factor. Defaults to `0.1`.
+- `τ_max`: if a step is rejected the new step size will get multiplied by factor, and this
+  parameter is the maximum value of that factor. Defaults to `0.5`.
+- `nexp`: the exponent of the loss, i.e. ``f_k=||F(x_k)||^{nexp}``. The paper uses
+  `nexp ∈ {1,2}`. Defaults to `2`.
+- `η_strategy`:  function to determine the parameter `η_k`, which enables growth
+  of ``||F||^2``. Called as ``η_k = η_strategy(f_1, k, x, F)`` with `f_1` initialized as
+  ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and
+  `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to
+  ``||F||^2 / k^2``.
 """
 struct SimpleDFSane{T} <: AbstractSimpleNonlinearSolveAlgorithm
     σ_min::T
-    σ_0::T
+    σ_max::T
+    σ_1::T
     M::Int
     γ::T
     τ_min::T
     τ_max::T
     nexp::Int
+    η_strategy::Function
 
-    function SimpleDFSane(; σ_min::Real = 1e-10, σ_0::Real = 1.0, M::Int = 10,
-                          γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
-                          nexp::Int = 2)
-        new{typeof(σ_min)}(σ_min, σ_0, M, γ, τ_min, τ_max, nexp)
+    function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
+                          M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+                          nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 / k^2)
+        new{typeof(σ_min)}(σ_min, σ_max, σ_1, M, γ, τ_min, τ_max, nexp, η_strategy)
     end
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem,
-                           alg::SimpleDFSane, args...; abstol = nothing,
-                           reltol = nothing,
-                           maxiters = 1000, kwargs...)
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
+                           args...; abstol = nothing, reltol = nothing, maxiters = 1000,
+                           kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     T = eltype(x)
     σ_min = float(alg.σ_min)
-    σ_k = float(alg.σ_0)
+    σ_max = float(alg.σ_max)
+    σ_k = float(alg.σ_1)
     M = alg.M
     γ = float(alg.γ)
     τ_min = float(alg.τ_min)
     τ_max = float(alg.τ_max)
     nexp = alg.nexp
+    η_strategy = alg.η_strategy
 
     if SciMLBase.isinplace(prob)
         error("SimpleDFSane currently only supports out-of-place nonlinear problems")
@@ -66,70 +89,67 @@ function SciMLBase.__solve(prob::NonlinearProblem,
     end
 
     f_k, F_k = ff(x)
-    α_0 = convert(T, 1.0)
-    f_0 = f_k
-    prev_fs = fill(f_k, M)
+    α_1 = convert(T, 1.0)
+    f_1 = f_k
+    history_f_k = fill(f_k, M)
 
     for k in 1:maxiters
         iszero(F_k) &&
             return SciMLBase.build_solution(prob, alg, x, F_k;
                                             retcode = ReturnCode.Success)
 
-        # Control spectral parameter
-        if abs(σ_k) > 1 / σ_min
-            σ_k = 1 / σ_min * sign(σ_k)
+        # Spectral parameter range check
+        if abs(σ_k) > σ_max
+            σ_k = sign(σ_k) * σ_max
         elseif abs(σ_k) < σ_min
-            σ_k = σ_min
+            σ_k = sign(σ_k) * σ_min
         end
 
         # Line search direction
         d = -σ_k * F_k
 
-        # Nonmonotone line search
-        η = f_0 / k^2
-
-        f_bar = maximum(prev_fs)
-        α_p = α_0
-        α_m = α_0
-        xp = x + α_p * d
-        fp, Fp = ff(xp)
+        η = η_strategy(f_1, k, x, F_k)
+        f̄ = maximum(history_f_k)
+        α_p = α_1
+        α_m = α_1
+        x_new = x + α_p * d
+        f_new, F_new = ff(x_new)
         while true
-            if fp ≤ f_bar + η - γ * α_p^2 * f_k
+            if f_new ≤ f̄ + η - γ * α_p^2 * f_k
                 break
             end
 
-            α_tp = α_p^2 * f_k / (fp + (2 * α_p - 1) * f_k)
-            xp = x - α_m * d
-            fp, Fp = ff(xp)
+            α_tp = α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
+            x_new = x - α_m * d
+            f_new, F_new = ff(x_new)
 
-            if fp ≤ f_bar + η - γ * α_m^2 * f_k
+            if f_new ≤ f̄ + η - γ * α_m^2 * f_k
                 break
             end
 
-            α_tm = α_m^2 * f_k / (fp + (2 * α_m - 1) * f_k)
+            α_tm = α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
             α_p = min(τ_max * α_p, max(α_tp, τ_min * α_p))
             α_m = min(τ_max * α_m, max(α_tm, τ_min * α_m))
-            xp = x + α_p * d
-            fp, Fp = ff(xp)
+            x_new = x + α_p * d
+            f_new, F_new = ff(x_new)
         end
 
-        if isapprox(xp, x, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, xp, Fp;
+        if isapprox(x_new, x, atol = atol, rtol = rtol)
+            return SciMLBase.build_solution(prob, alg, x_new, F_new;
                                             retcode = ReturnCode.Success)
         end
         # Update spectral parameter
-        s_k = xp - x
-        y_k = Fp - F_k
-        σ_k = dot(s_k, s_k) / dot(s_k, y_k)
+        s_k = x_new - x
+        y_k = F_new - F_k
+        σ_k = (s_k' * s_k) / (s_k' * y_k)
 
         # Take step
-        x = xp
-        F_k = Fp
-        f_k = fp
+        x = x_new
+        F_k = F_new
+        f_k = f_new
 
         # Store function value
-        idx_to_replace = k % M + 1
-        prev_fs[idx_to_replace] = fp
+        history_f_k[k % M + 1] = f_new
     end
     return SciMLBase.build_solution(prob, alg, x, F_k; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 782c26625..d8fb583f4 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -160,7 +160,8 @@ for alg in [Bisection(), Falsi(), Ridder(), Brent()]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
 
-for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion(), SimpleDFSane())
+for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion(),
+            SimpleDFSane())
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -313,45 +314,57 @@ for options in list_of_options
     @test all(abs.(f(u, p)) .< 1e-10)
 end
 
-# # Test that `SimpleDFSane` passes a test that `SimpleNewtonRaphson` fails on.
-# u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
-# global g, f
-# f = (u, p) -> 0.010000000000000002 .+
-#               10.000000000000002 ./ (1 .+
-#                (0.21640425613334457 .+
-#                 216.40425613334457 ./ (1 .+
-#                  (0.21640425613334457 .+
-#                   216.40425613334457 ./
-#                   (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
-#               0.0011552453009332421u .- p
-# g = function (p)
-#     probN = NonlinearProblem{false}(f, u0, p)
-#     sol = solve(probN, SimpleDFSane())
-#     return sol.u
-# end
-# p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
-# u = g(p)
-# f(u, p)
-# @test all(abs.(f(u, p)) .< 1e-10)
-
-# # Test kwars in `SimpleDFSane`
-
-
-# list_of_options = zip(max_trust_radius, initial_trust_radius, step_threshold,
-#                       shrink_threshold, expand_threshold, shrink_factor,
-#                       expand_factor, max_shrink_times)
-# for options in list_of_options
-#     local probN, sol, alg
-#     alg = SimpleDFSane(max_trust_radius = options[1],
-#                             initial_trust_radius = options[2],
-#                             step_threshold = options[3],
-#                             shrink_threshold = options[4],
-#                             expand_threshold = options[5],
-#                             shrink_factor = options[6],
-#                             expand_factor = options[7],
-#                             max_shrink_times = options[8])
-
-#     probN = NonlinearProblem(f, u0, p)
-#     sol = solve(probN, alg)
-#     @test all(abs.(f(u, p)) .< 1e-10)
-# end
+# Test that `SimpleDFSane` passes a test that `SimpleNewtonRaphson` fails on.
+u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
+global g, f
+f = (u, p) -> 0.010000000000000002 .+
+              10.000000000000002 ./ (1 .+
+               (0.21640425613334457 .+
+                216.40425613334457 ./ (1 .+
+                 (0.21640425613334457 .+
+                  216.40425613334457 ./
+                  (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
+              0.0011552453009332421u .- p
+g = function (p)
+    probN = NonlinearProblem{false}(f, u0, p)
+    sol = solve(probN, SimpleDFSane())
+    return sol.u
+end
+p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+u = g(p)
+f(u, p)
+@test all(abs.(f(u, p)) .< 1e-10)
+
+# Test kwars in `SimpleDFSane`
+σ_min = [1e-10, 1e-5, 1e-4]
+σ_max = [1e10, 1e5, 1e4]
+σ_1 = [1.0, 0.5, 2.0]
+M = [10, 1, 100]
+γ = [1e-4, 1e-3, 1e-5]
+τ_min = [0.1, 0.2, 0.3]
+τ_max = [0.5, 0.8, 0.9]
+nexp = [2, 1, 2]
+η_strategy = [
+    (f_1, k, x, F) -> f_1 / k^2,
+    (f_1, k, x, F) -> f_1 / k^3,
+    (f_1, k, x, F) -> f_1 / k^4,
+]
+
+list_of_options = zip(σ_min, σ_max, σ_1, M, γ, τ_min, τ_max, nexp,
+                      η_strategy)
+for options in list_of_options
+    local probN, sol, alg
+    alg = SimpleDFSane(σ_min = options[1],
+                       σ_max = options[2],
+                       σ_1 = options[3],
+                       M = options[4],
+                       γ = options[5],
+                       τ_min = options[6],
+                       τ_max = options[7],
+                       nexp = options[8],
+                       η_strategy = options[9])
+
+    probN = NonlinearProblem(f, u0, p)
+    sol = solve(probN, alg)
+    @test all(abs.(f(u, p)) .< 1e-10)
+end

From d8c13cfa26771e468220871f2f506220bf0952a2 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 6 Feb 2023 16:32:37 -0500
Subject: [PATCH 064/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 3b8b5d859..2a953148a 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.9"
+version = "0.1.10"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"

From 4199b52bb66f81e8680500663a9ffa7bbfbe7134 Mon Sep 17 00:00:00 2001
From: Simon Carlson <ccsimon123@gmail.com>
Date: Tue, 7 Feb 2023 13:01:28 +0100
Subject: [PATCH 065/375] Dead links in Trust Region fix

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 329eb8808..edd143215 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -13,9 +13,7 @@ SimpleTrustRegion(; chunk_size = Val{0}(),
                     max_shrink_times::Int = 32
 ```
 
-A low-overhead implementation of a
-[trust-region](https://optimization.mccormick.northwestern.edu/index.php/Trust-region_methods)
-solver
+A low-overhead implementation of a trust-region solver.
 
 ### Keyword Arguments
 
@@ -39,12 +37,12 @@ solver
   compared with a value `r`, which is the actual reduction in the objective function divided
   by the predicted reduction. If `step_threshold > r` the model is not a good approximation,
   and the step is rejected. Defaults to `0.1`. For more details, see
-  [Trust-region methods](https://optimization.mccormick.northwestern.edu/index.php/Trust-region_methods)
+  [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4)
 - `shrink_threshold`: the threshold for shrinking the trust region radius. In every
   iteration, the threshold is compared with a value `r` which is the actual reduction in the
   objective function divided by the predicted reduction. If `shrink_threshold > r` the trust
   region radius is shrunk by `shrink_factor`. Defaults to `0.25`. For more details, see
-  [Trust-region methods](https://optimization.mccormick.northwestern.edu/index.php/Trust-region_methods)
+  [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4)
 - `expand_threshold`: the threshold for expanding the trust region radius. If a step is
   taken, i.e `step_threshold < r` (with `r` defined in `shrink_threshold`), a check is also
   made to see if `expand_threshold < r`. If that is true, the trust region radius is

From 50d348f89b5e851b6b7db3055b9dc91852a270da Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 10 Feb 2023 16:32:42 -0500
Subject: [PATCH 066/375] Add limited memory broyden implementation

---
 .../src/SimpleNonlinearSolve.jl               |  5 +-
 lib/SimpleNonlinearSolve/src/lbroyden.jl      | 83 +++++++++++++++++++
 2 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/lbroyden.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 940db0835..5b170ce6c 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -20,6 +20,7 @@ include("bisection.jl")
 include("falsi.jl")
 include("raphson.jl")
 include("broyden.jl")
+include("lbroyden.jl")
 include("klement.jl")
 include("trustRegion.jl")
 include("ridder.jl")
@@ -52,7 +53,7 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
 end end
 
 # DiffEq styled algorithms
-export Bisection, Brent, Broyden, SimpleDFSane, Falsi, Klement, Ridder, SimpleNewtonRaphson,
-       SimpleTrustRegion
+export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Klement,
+       Ridder, SimpleNewtonRaphson, SimpleTrustRegion
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
new file mode 100644
index 000000000..61423d8ce
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -0,0 +1,83 @@
+Base.@kwdef struct LBroyden <: AbstractSimpleNonlinearSolveAlgorithm
+    threshold::Int = 27
+end
+
+@views function SciMLBase.__solve(prob::NonlinearProblem,
+                                  alg::LBroyden, args...; abstol = nothing,
+                                  reltol = nothing,
+                                  maxiters = 1000, kwargs...)
+    threshold = min(maxiters, alg.threshold)
+    x = float(prob.u0)
+
+    if x isa Number
+        restore_scalar = true
+        x = [x]
+        f = u -> prob.f(u[], prob.p)
+    else
+        f = Base.Fix2(prob.f, prob.p)
+        restore_scalar = false
+    end
+
+    fₙ = f(x)
+    T = eltype(x)
+
+    if SciMLBase.isinplace(prob)
+        error("LBroyden currently only supports out-of-place nonlinear problems")
+    end
+
+    U = fill!(similar(x, (threshold, length(x))), zero(T))
+    Vᵀ = fill!(similar(x, (length(x), threshold)), zero(T))
+
+    atol = abstol !== nothing ? abstol :
+           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
+    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+
+    xₙ = x
+    xₙ₋₁ = x
+    fₙ₋₁ = fₙ
+    update = fₙ
+    for i in 1:maxiters
+        xₙ = xₙ₋₁ .+ update
+        fₙ = f(xₙ)
+        Δxₙ = xₙ .- xₙ₋₁
+        Δfₙ = fₙ .- fₙ₋₁
+
+        if iszero(fₙ)
+            xₙ = restore_scalar ? xₙ[] : xₙ
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
+        end
+
+        if isapprox(xₙ, xₙ₋₁; atol, rtol)
+            xₙ = restore_scalar ? xₙ[] : xₙ
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
+        end
+
+        _U = U[1:min(threshold, i), :]
+        _Vᵀ = Vᵀ[:, 1:min(threshold, i)]
+
+        vᵀ = _rmatvec(_U, _Vᵀ, Δxₙ)
+        mvec = _matvec(_U, _Vᵀ, Δfₙ)
+        Δxₙ = (Δxₙ .- mvec) ./ (sum(vᵀ .* Δfₙ) .+ eps(T))
+
+        Vᵀ[:, mod1(i, threshold)] .= vᵀ
+        U[mod1(i, threshold), :] .= Δxₙ
+
+        update = -_matvec(U[1:min(threshold, i + 1), :], Vᵀ[:, 1:min(threshold, i + 1)], fₙ)
+
+        xₙ₋₁ = xₙ
+        fₙ₋₁ = fₙ
+    end
+
+    xₙ = restore_scalar ? xₙ[] : xₙ
+    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+end
+
+function _rmatvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
+                  x::Union{<:AbstractVector, <:Number})
+    return -x .+ dropdims(sum(U .* sum(Vᵀ .* x; dims = 1)'; dims = 1); dims = 1)
+end
+
+function _matvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
+                 x::Union{<:AbstractVector, <:Number})
+    return -x .+ dropdims(sum(sum(x .* U'; dims = 1) .* Vᵀ; dims = 2); dims = 2)
+end

From 87ea1bcec19460971ce8b74047d5d4e2e1d1a3f8 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 10 Feb 2023 16:58:47 -0500
Subject: [PATCH 067/375] Add some tests

---
 lib/SimpleNonlinearSolve/Project.toml       | 2 +-
 lib/SimpleNonlinearSolve/src/lbroyden.jl    | 8 +++++++-
 lib/SimpleNonlinearSolve/test/basictests.jl | 8 +++++---
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 2a953148a..6288c3ecc 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.10"
+version = "0.1.11"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
index 61423d8ce..f983bbd66 100644
--- a/lib/SimpleNonlinearSolve/src/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -1,3 +1,9 @@
+"""
+    LBroyden(threshold::Int = 27)
+
+A limited memory implementation of Broyden. This method applies the L-BFGS scheme to
+Broyden's method.
+"""
 Base.@kwdef struct LBroyden <: AbstractSimpleNonlinearSolveAlgorithm
     threshold::Int = 27
 end
@@ -57,7 +63,7 @@ end
 
         vᵀ = _rmatvec(_U, _Vᵀ, Δxₙ)
         mvec = _matvec(_U, _Vᵀ, Δfₙ)
-        Δxₙ = (Δxₙ .- mvec) ./ (sum(vᵀ .* Δfₙ) .+ eps(T))
+        Δxₙ = (Δxₙ .- mvec) ./ (sum(vᵀ .* Δfₙ) .+ convert(T, 1e-5))
 
         Vᵀ[:, mod1(i, threshold)] .= vᵀ
         U[mod1(i, threshold), :] .= Δxₙ
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index d8fb583f4..ad83fd224 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -78,7 +78,7 @@ using ForwardDiff
 # Immutable
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
-for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion(),
+for alg in (SimpleNewtonRaphson(), Broyden(), LBroyden(), Klement(), SimpleTrustRegion(),
             SimpleDFSane())
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
@@ -94,7 +94,7 @@ end
 
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
-for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion(),
+for alg in (SimpleNewtonRaphson(), Broyden(), LBroyden(), Klement(), SimpleTrustRegion(),
             SimpleDFSane())
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
@@ -160,7 +160,7 @@ for alg in [Bisection(), Falsi(), Ridder(), Brent()]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
 
-for alg in (SimpleNewtonRaphson(), Broyden(), Klement(), SimpleTrustRegion(),
+for alg in (SimpleNewtonRaphson(), Broyden(), LBroyden(), Klement(), SimpleTrustRegion(),
             SimpleDFSane())
     global g, p
     g = function (p)
@@ -181,6 +181,7 @@ probN = NonlinearProblem(f, u0)
 @test solve(probN, SimpleTrustRegion()).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleTrustRegion(; autodiff = false)).u[end] ≈ sqrt(2.0)
 @test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
+@test solve(probN, LBroyden()).u[end] ≈ sqrt(2.0)
 @test solve(probN, Klement()).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleDFSane()).u[end] ≈ sqrt(2.0)
 
@@ -199,6 +200,7 @@ for u0 in [1.0, [1, 1.0]]
     @test solve(probN, SimpleTrustRegion(; autodiff = false)).u ≈ sol
 
     @test solve(probN, Broyden()).u ≈ sol
+    @test solve(probN, LBroyden()).u ≈ sol
     @test solve(probN, Klement()).u ≈ sol
     @test solve(probN, SimpleDFSane()).u ≈ sol
 end

From 775005604e2bd31c7215c3b0217b524903a7be08 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 11 Feb 2023 08:56:52 -0500
Subject: [PATCH 068/375] Update basictests.jl

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index ad83fd224..3f386d276 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -166,7 +166,7 @@ for alg in (SimpleNewtonRaphson(), Broyden(), LBroyden(), Klement(), SimpleTrust
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
         sol = solve(probN, alg)
-        return [sol.u]
+        return [abs(sol.u)]
     end
     @test g(p) ≈ [sqrt(p[2] / p[1])]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)

From 245a6d7be9332fc1ef195889b0f9dfc78a9ab045 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 12 Feb 2023 13:02:27 -0500
Subject: [PATCH 069/375] Make broyden batched

---
 lib/SimpleNonlinearSolve/Project.toml         |  4 ++-
 .../src/SimpleNonlinearSolve.jl               |  1 +
 lib/SimpleNonlinearSolve/src/broyden.jl       | 29 +++++++++++++------
 lib/SimpleNonlinearSolve/src/lbroyden.jl      |  7 ++---
 lib/SimpleNonlinearSolve/src/utils.jl         | 28 ++++++++++++++----
 5 files changed, 50 insertions(+), 19 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 6288c3ecc..351bb62a4 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.11"
+version = "0.1.12"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
@@ -9,6 +9,7 @@ DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
@@ -19,6 +20,7 @@ ArrayInterfaceCore = "0.1.1"
 DiffEqBase = "6.114"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
+NNlib = "0.8"
 Reexport = "0.2, 1"
 SciMLBase = "1.73"
 SnoopPrecompile = "1"
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 5b170ce6c..b48e33ed2 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -7,6 +7,7 @@ using StaticArraysCore
 using LinearAlgebra
 import ArrayInterfaceCore
 using DiffEqBase
+using NNlib  # Batched Matrix Multiplication
 
 @reexport using SciMLBase
 
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index a05caa00a..7539a2bfe 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -8,15 +8,18 @@ and static array problems.
 """
 struct Broyden <: AbstractSimpleNonlinearSolveAlgorithm end
 
-function SciMLBase.__solve(prob::NonlinearProblem,
-                           alg::Broyden, args...; abstol = nothing,
-                           reltol = nothing,
-                           maxiters = 1000, kwargs...)
+function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...; abstol = nothing,
+                           reltol = nothing, maxiters = 1000, batch = false, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
+
+    if batch && ndims(x) != 2
+        error("`batch` mode works only if `ndims(prob.u0) == 2`")
+    end
+
     fₙ = f(x)
     T = eltype(x)
-    J⁻¹ = init_J(x)
+    J⁻¹ = init_J(x; batch)
 
     if SciMLBase.isinplace(prob)
         error("Broyden currently only supports out-of-place nonlinear problems")
@@ -30,11 +33,14 @@ function SciMLBase.__solve(prob::NonlinearProblem,
     xₙ₋₁ = x
     fₙ₋₁ = fₙ
     for _ in 1:maxiters
-        xₙ = xₙ₋₁ - J⁻¹ * fₙ₋₁
+        xₙ = xₙ₋₁ .- _batched_mul(J⁻¹, fₙ₋₁, batch)
         fₙ = f(xₙ)
-        Δxₙ = xₙ - xₙ₋₁
-        Δfₙ = fₙ - fₙ₋₁
-        J⁻¹ += ((Δxₙ - J⁻¹ * Δfₙ) ./ (Δxₙ' * J⁻¹ * Δfₙ)) * (Δxₙ' * J⁻¹)
+        Δxₙ = xₙ .- xₙ₋₁
+        Δfₙ = fₙ .- fₙ₋₁
+        J⁻¹Δfₙ = _batched_mul(J⁻¹, Δfₙ, batch)
+        J⁻¹ += _batched_mul(((Δxₙ .- J⁻¹Δfₙ, batch) ./
+                             (_batched_mul(_batch_transpose(Δxₙ, batch), J⁻¹Δfₙ, batch))),
+                            _batched_mul(_batch_transpose(Δxₙ, batch), J⁻¹, batch), batch)
 
         iszero(fₙ) &&
             return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
@@ -50,3 +56,8 @@ function SciMLBase.__solve(prob::NonlinearProblem,
 
     return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
 end
+
+function _batch_transpose(x, batch)
+    !batch && return x'
+    return reshape(x, 1, size(x)...)
+end
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
index f983bbd66..f8ed9f5d3 100644
--- a/lib/SimpleNonlinearSolve/src/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -8,10 +8,9 @@ Base.@kwdef struct LBroyden <: AbstractSimpleNonlinearSolveAlgorithm
     threshold::Int = 27
 end
 
-@views function SciMLBase.__solve(prob::NonlinearProblem,
-                                  alg::LBroyden, args...; abstol = nothing,
-                                  reltol = nothing,
-                                  maxiters = 1000, kwargs...)
+@views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden, args...;
+                                  abstol = nothing, reltol = nothing, maxiters = 1000,
+                                  batch = false, kwargs...)
     threshold = min(maxiters, alg.threshold)
     x = float(prob.u0)
 
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index f3b7de9f6..6f12b829f 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -34,14 +34,17 @@ value(x) = x
 value(x::Dual) = ForwardDiff.value(x)
 value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
 
-function init_J(x)
-    J = ArrayInterfaceCore.zeromatrix(x)
-    if ismutable(x)
-        J[diagind(J)] .= one(eltype(x))
+function init_J(x; batch = false)
+    x_ = batch ? x[:, 1] : x
+
+    J = ArrayInterfaceCore.zeromatrix(x_)
+    if ismutable(x_)
+        J[diagind(J)] .= one(eltype(x_))
     else
         J += I
     end
-    return J
+
+    return batch ? repeat(J, 1, 1, size(x, 2)) : J
 end
 
 function dogleg_method(H, g, Δ)
@@ -68,3 +71,18 @@ function dogleg_method(H, g, Δ)
     tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd
     return δsd + tau * δN_δsd
 end
+
+_batched_mul(x, y, batch) = x * y
+function _batched_mul(x::AbstractArray{T, 3}, y::AbstractMatrix, batch) where {T}
+    !batch && return x * y
+    return dropdims(batched_mul(x, reshape(y, size(y, 1), 1, size(y, 2))); dims = 2)
+end
+function _batched_mul(x::AbstractMatrix, y::AbstractArray{T, 3}, batch) where {T}
+    !batch && return x * y
+    return batched_mul(reshape(x, size(x, 1), 1, size(x, 2)), y)
+end
+function _batched_mul(x::AbstractArray{T1, 3}, y::AbstractArray{T2, 3},
+                      batch) where {T1, T2}
+    !batch && return x * y
+    return batched_mul(x, y)
+end
\ No newline at end of file

From 3bd1c85c9ad148a7b86d32b9d15f55951e0233c8 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 13 Feb 2023 11:07:39 -0500
Subject: [PATCH 070/375] Make it an extension

---
 lib/SimpleNonlinearSolve/Project.toml         |  7 ++-
 .../SimpleBatchedNonlinearSolveExt.jl         |  9 ++++
 .../SimpleBatchedNonlinearSolveExt/broyden.jl | 48 +++++++++++++++++++
 .../lbroyden.jl                               |  0
 .../SimpleBatchedNonlinearSolveExt/utils.jl   | 25 ++++++++++
 .../src/SimpleNonlinearSolve.jl               |  1 -
 lib/SimpleNonlinearSolve/src/broyden.jl       | 38 +++++++--------
 lib/SimpleNonlinearSolve/src/utils.jl         | 28 ++---------
 8 files changed, 109 insertions(+), 47 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/SimpleBatchedNonlinearSolveExt.jl
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/broyden.jl
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/lbroyden.jl
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/utils.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 351bb62a4..4852c1f78 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -9,12 +9,17 @@ DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
+[weakdeps]
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+
+[extensions]
+SimpleBatchedNonlinearSolveExt = "NNlib"
+
 [compat]
 ArrayInterfaceCore = "0.1.1"
 DiffEqBase = "6.114"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/SimpleBatchedNonlinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/SimpleBatchedNonlinearSolveExt.jl
new file mode 100644
index 000000000..5a7483e3f
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/SimpleBatchedNonlinearSolveExt.jl
@@ -0,0 +1,9 @@
+module SimpleBatchedNonlinearSolveExt
+
+using SimpleNonlinearSolve, SciMLBase, NNlib
+
+include("utils.jl")
+include("broyden.jl")
+include("lbroyden.jl")
+
+end
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/broyden.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/broyden.jl
new file mode 100644
index 000000000..2476e9570
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/broyden.jl
@@ -0,0 +1,48 @@
+function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
+                           abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    x = float(prob.u0)
+
+    if ndims(x) != 2
+        error("`batch` mode works only if `ndims(prob.u0) == 2`")
+    end
+
+    fₙ = f(x)
+    T = eltype(x)
+    J⁻¹ = _init_J_batched(x)
+
+    if SciMLBase.isinplace(prob)
+        error("Broyden currently only supports out-of-place nonlinear problems")
+    end
+
+    atol = abstol !== nothing ? abstol :
+           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
+    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+
+    xₙ = x
+    xₙ₋₁ = x
+    fₙ₋₁ = fₙ
+    for _ in 1:maxiters
+        xₙ = xₙ₋₁ .- _batched_mul(J⁻¹, fₙ₋₁, batch)
+        fₙ = f(xₙ)
+        Δxₙ = xₙ .- xₙ₋₁
+        Δfₙ = fₙ .- fₙ₋₁
+        J⁻¹Δfₙ = _batched_mul(J⁻¹, Δfₙ, batch)
+        J⁻¹ += _batched_mul(((Δxₙ .- J⁻¹Δfₙ, batch) ./
+                             (_batched_mul(_batch_transpose(Δxₙ, batch), J⁻¹Δfₙ, batch))),
+                            _batched_mul(_batch_transpose(Δxₙ, batch), J⁻¹, batch), batch)
+
+        iszero(fₙ) &&
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                            retcode = ReturnCode.Success)
+
+        if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
+                                            retcode = ReturnCode.Success)
+        end
+        xₙ₋₁ = xₙ
+        fₙ₋₁ = fₙ
+    end
+
+    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/lbroyden.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/lbroyden.jl
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/utils.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/utils.jl
new file mode 100644
index 000000000..4dfd4f34e
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/utils.jl
@@ -0,0 +1,25 @@
+_batch_transpose(x) = reshape(x, 1, size(x)...)
+
+_batched_mul(x, y) = x * y
+
+function _batched_mul(x::AbstractArray{T, 3}, y::AbstractMatrix) where {T}
+    return dropdims(batched_mul(x, reshape(y, size(y, 1), 1, size(y, 2))); dims = 2)
+end
+
+function _batched_mul(x::AbstractMatrix, y::AbstractArray{T, 3}) where {T}
+    return batched_mul(reshape(x, size(x, 1), 1, size(x, 2)), y)
+end
+
+function _batched_mul(x::AbstractArray{T1, 3}, y::AbstractArray{T2, 3}) where {T1, T2}
+    return batched_mul(x, y)
+end
+
+function _init_J_batched(x::AbstractMatrix{T}) where {T}
+    J = ArrayInterfaceCore.zeromatrix(x[:, 1])
+    if ismutable(x)
+        J[diagind(J)] .= one(eltype(x))
+    else
+        J += I
+    end
+    return repeat(J, 1, 1, size(x, 2))
+end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index b48e33ed2..5b170ce6c 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -7,7 +7,6 @@ using StaticArraysCore
 using LinearAlgebra
 import ArrayInterfaceCore
 using DiffEqBase
-using NNlib  # Batched Matrix Multiplication
 
 @reexport using SciMLBase
 
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 7539a2bfe..7820be8e9 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -1,25 +1,26 @@
 """
-```julia
-Broyden()
-```
+    Broyden()
 
 A low-overhead implementation of Broyden. This method is non-allocating on scalar
 and static array problems.
 """
-struct Broyden <: AbstractSimpleNonlinearSolveAlgorithm end
+struct Broyden{batched} <: AbstractSimpleNonlinearSolveAlgorithm
+    Broyden(batched = false) = new{batched}()
+end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...; abstol = nothing,
-                           reltol = nothing, maxiters = 1000, batch = false, kwargs...)
+function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{false}, args...;
+                           abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
 
-    if batch && ndims(x) != 2
-        error("`batch` mode works only if `ndims(prob.u0) == 2`")
-    end
+    # if batch && ndims(x) != 2
+    #     error("`batch` mode works only if `ndims(prob.u0) == 2`")
+    # end
 
     fₙ = f(x)
     T = eltype(x)
-    J⁻¹ = init_J(x; batch)
+    # J⁻¹ = init_J(x; batch)
+    J⁻¹ = init_J(x)
 
     if SciMLBase.isinplace(prob)
         error("Broyden currently only supports out-of-place nonlinear problems")
@@ -33,14 +34,12 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...; abstol
     xₙ₋₁ = x
     fₙ₋₁ = fₙ
     for _ in 1:maxiters
-        xₙ = xₙ₋₁ .- _batched_mul(J⁻¹, fₙ₋₁, batch)
+        xₙ = xₙ₋₁ - J⁻¹ * fₙ₋₁
         fₙ = f(xₙ)
-        Δxₙ = xₙ .- xₙ₋₁
-        Δfₙ = fₙ .- fₙ₋₁
-        J⁻¹Δfₙ = _batched_mul(J⁻¹, Δfₙ, batch)
-        J⁻¹ += _batched_mul(((Δxₙ .- J⁻¹Δfₙ, batch) ./
-                             (_batched_mul(_batch_transpose(Δxₙ, batch), J⁻¹Δfₙ, batch))),
-                            _batched_mul(_batch_transpose(Δxₙ, batch), J⁻¹, batch), batch)
+        Δxₙ = xₙ - xₙ₋₁
+        Δfₙ = fₙ - fₙ₋₁
+        J⁻¹Δfₙ = J⁻¹ * Δfₙ
+        J⁻¹ += ((Δxₙ .- J⁻¹Δfₙ) ./ (Δxₙ' * J⁻¹Δfₙ)) * (Δxₙ' * J⁻¹)
 
         iszero(fₙ) &&
             return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
@@ -56,8 +55,3 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...; abstol
 
     return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
 end
-
-function _batch_transpose(x, batch)
-    !batch && return x'
-    return reshape(x, 1, size(x)...)
-end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 6f12b829f..f3b7de9f6 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -34,17 +34,14 @@ value(x) = x
 value(x::Dual) = ForwardDiff.value(x)
 value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
 
-function init_J(x; batch = false)
-    x_ = batch ? x[:, 1] : x
-
-    J = ArrayInterfaceCore.zeromatrix(x_)
-    if ismutable(x_)
-        J[diagind(J)] .= one(eltype(x_))
+function init_J(x)
+    J = ArrayInterfaceCore.zeromatrix(x)
+    if ismutable(x)
+        J[diagind(J)] .= one(eltype(x))
     else
         J += I
     end
-
-    return batch ? repeat(J, 1, 1, size(x, 2)) : J
+    return J
 end
 
 function dogleg_method(H, g, Δ)
@@ -71,18 +68,3 @@ function dogleg_method(H, g, Δ)
     tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd
     return δsd + tau * δN_δsd
 end
-
-_batched_mul(x, y, batch) = x * y
-function _batched_mul(x::AbstractArray{T, 3}, y::AbstractMatrix, batch) where {T}
-    !batch && return x * y
-    return dropdims(batched_mul(x, reshape(y, size(y, 1), 1, size(y, 2))); dims = 2)
-end
-function _batched_mul(x::AbstractMatrix, y::AbstractArray{T, 3}, batch) where {T}
-    !batch && return x * y
-    return batched_mul(reshape(x, size(x, 1), 1, size(x, 2)), y)
-end
-function _batched_mul(x::AbstractArray{T1, 3}, y::AbstractArray{T2, 3},
-                      batch) where {T1, T2}
-    !batch && return x * y
-    return batched_mul(x, y)
-end
\ No newline at end of file

From 33cf8b36ac746c877e492cd67bd0cc3f28cc9ced Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 15 Feb 2023 11:28:44 -0500
Subject: [PATCH 071/375] Add requires for backward compat

---
 lib/SimpleNonlinearSolve/Project.toml         |  4 ++-
 ...n.jl => SimpleBatchedNonlinearSolveExt.jl} | 34 +++++++++++++++++++
 .../SimpleBatchedNonlinearSolveExt.jl         |  9 -----
 .../lbroyden.jl                               |  0
 .../SimpleBatchedNonlinearSolveExt/utils.jl   | 25 --------------
 .../src/SimpleNonlinearSolve.jl               | 12 +++++++
 lib/SimpleNonlinearSolve/src/broyden.jl       |  9 ++---
 7 files changed, 51 insertions(+), 42 deletions(-)
 rename lib/SimpleNonlinearSolve/ext/{SimpleBatchedNonlinearSolveExt/broyden.jl => SimpleBatchedNonlinearSolveExt.jl} (66%)
 delete mode 100644 lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/SimpleBatchedNonlinearSolveExt.jl
 delete mode 100644 lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/lbroyden.jl
 delete mode 100644 lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/utils.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 4852c1f78..28e3410a9 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -10,6 +10,7 @@ FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
+Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
@@ -34,10 +35,11 @@ julia = "1.6"
 
 [extras]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays"]
+test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib"]
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/broyden.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
similarity index 66%
rename from lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/broyden.jl
rename to lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
index 2476e9570..88de48871 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/broyden.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
@@ -1,3 +1,34 @@
+module SimpleBatchedNonlinearSolveExt
+
+using SimpleNonlinearSolve, SciMLBase
+isdefined(Base, :get_extension) ? (using NNlib) : (using ..NNlib)
+
+_batch_transpose(x) = reshape(x, 1, size(x)...)
+
+_batched_mul(x, y) = x * y
+
+function _batched_mul(x::AbstractArray{T, 3}, y::AbstractMatrix) where {T}
+    return dropdims(batched_mul(x, reshape(y, size(y, 1), 1, size(y, 2))); dims = 2)
+end
+
+function _batched_mul(x::AbstractMatrix, y::AbstractArray{T, 3}) where {T}
+    return batched_mul(reshape(x, size(x, 1), 1, size(x, 2)), y)
+end
+
+function _batched_mul(x::AbstractArray{T1, 3}, y::AbstractArray{T2, 3}) where {T1, T2}
+    return batched_mul(x, y)
+end
+
+function _init_J_batched(x::AbstractMatrix{T}) where {T}
+    J = ArrayInterfaceCore.zeromatrix(x[:, 1])
+    if ismutable(x)
+        J[diagind(J)] .= one(eltype(x))
+    else
+        J += I
+    end
+    return repeat(J, 1, 1, size(x, 2))
+end
+
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
                            abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
@@ -46,3 +77,6 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
 
     return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
 end
+
+
+end
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/SimpleBatchedNonlinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/SimpleBatchedNonlinearSolveExt.jl
deleted file mode 100644
index 5a7483e3f..000000000
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/SimpleBatchedNonlinearSolveExt.jl
+++ /dev/null
@@ -1,9 +0,0 @@
-module SimpleBatchedNonlinearSolveExt
-
-using SimpleNonlinearSolve, SciMLBase, NNlib
-
-include("utils.jl")
-include("broyden.jl")
-include("lbroyden.jl")
-
-end
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/lbroyden.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/lbroyden.jl
deleted file mode 100644
index e69de29bb..000000000
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/utils.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/utils.jl
deleted file mode 100644
index 4dfd4f34e..000000000
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt/utils.jl
+++ /dev/null
@@ -1,25 +0,0 @@
-_batch_transpose(x) = reshape(x, 1, size(x)...)
-
-_batched_mul(x, y) = x * y
-
-function _batched_mul(x::AbstractArray{T, 3}, y::AbstractMatrix) where {T}
-    return dropdims(batched_mul(x, reshape(y, size(y, 1), 1, size(y, 2))); dims = 2)
-end
-
-function _batched_mul(x::AbstractMatrix, y::AbstractArray{T, 3}) where {T}
-    return batched_mul(reshape(x, size(x, 1), 1, size(x, 2)), y)
-end
-
-function _batched_mul(x::AbstractArray{T1, 3}, y::AbstractArray{T2, 3}) where {T1, T2}
-    return batched_mul(x, y)
-end
-
-function _init_J_batched(x::AbstractMatrix{T}) where {T}
-    J = ArrayInterfaceCore.zeromatrix(x[:, 1])
-    if ismutable(x)
-        J[diagind(J)] .= one(eltype(x))
-    else
-        J += I
-    end
-    return repeat(J, 1, 1, size(x, 2))
-end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 5b170ce6c..51785e323 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -10,6 +10,18 @@ using DiffEqBase
 
 @reexport using SciMLBase
 
+if !isdefined(Base, :get_extension)
+    using Requires
+end
+
+function __init__()
+    @static if !isdefined(Base, :get_extension)
+        @require NNlib="872c559c-99b0-510c-b3b7-b6c96a88d5cd" begin
+            include("../ext/SimpleBatchedNonlinearSolveExt.jl")
+        end
+    end
+end
+
 abstract type AbstractSimpleNonlinearSolveAlgorithm <: SciMLBase.AbstractNonlinearAlgorithm end
 abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 abstract type AbstractNewtonAlgorithm{CS, AD, FDT} <: AbstractSimpleNonlinearSolveAlgorithm end
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 7820be8e9..15da8a77e 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -1,11 +1,11 @@
 """
-    Broyden()
+    Broyden(; batched = false)
 
 A low-overhead implementation of Broyden. This method is non-allocating on scalar
 and static array problems.
 """
 struct Broyden{batched} <: AbstractSimpleNonlinearSolveAlgorithm
-    Broyden(batched = false) = new{batched}()
+    Broyden(; batched = false) = new{batched}()
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{false}, args...;
@@ -13,13 +13,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{false}, args...;
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
 
-    # if batch && ndims(x) != 2
-    #     error("`batch` mode works only if `ndims(prob.u0) == 2`")
-    # end
-
     fₙ = f(x)
     T = eltype(x)
-    # J⁻¹ = init_J(x; batch)
     J⁻¹ = init_J(x)
 
     if SciMLBase.isinplace(prob)

From e88d64dd60c95b5cba5c29786363db08aa0a6ccd Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 16 Feb 2023 14:51:45 -0500
Subject: [PATCH 072/375] Add tests for batched broyden

---
 lib/SimpleNonlinearSolve/.gitignore             |  3 +++
 .../ext/SimpleBatchedNonlinearSolveExt.jl       | 17 ++++++++---------
 .../src/SimpleNonlinearSolve.jl                 |  4 +---
 lib/SimpleNonlinearSolve/src/broyden.jl         |  5 +++++
 lib/SimpleNonlinearSolve/test/basictests.jl     | 12 ++++++++++++
 5 files changed, 29 insertions(+), 12 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/.gitignore

diff --git a/lib/SimpleNonlinearSolve/.gitignore b/lib/SimpleNonlinearSolve/.gitignore
new file mode 100644
index 000000000..e4cfbfe81
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/.gitignore
@@ -0,0 +1,3 @@
+Manifest.toml
+
+wip
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
index 88de48871..d59970471 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
@@ -1,6 +1,6 @@
 module SimpleBatchedNonlinearSolveExt
 
-using SimpleNonlinearSolve, SciMLBase
+using ArrayInterfaceCore, LinearAlgebra, SimpleNonlinearSolve, SciMLBase
 isdefined(Base, :get_extension) ? (using NNlib) : (using ..NNlib)
 
 _batch_transpose(x) = reshape(x, 1, size(x)...)
@@ -53,15 +53,15 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
     xₙ = x
     xₙ₋₁ = x
     fₙ₋₁ = fₙ
-    for _ in 1:maxiters
-        xₙ = xₙ₋₁ .- _batched_mul(J⁻¹, fₙ₋₁, batch)
+    for i in 1:maxiters
+        xₙ = xₙ₋₁ .- _batched_mul(J⁻¹, fₙ₋₁)
         fₙ = f(xₙ)
         Δxₙ = xₙ .- xₙ₋₁
         Δfₙ = fₙ .- fₙ₋₁
-        J⁻¹Δfₙ = _batched_mul(J⁻¹, Δfₙ, batch)
-        J⁻¹ += _batched_mul(((Δxₙ .- J⁻¹Δfₙ, batch) ./
-                             (_batched_mul(_batch_transpose(Δxₙ, batch), J⁻¹Δfₙ, batch))),
-                            _batched_mul(_batch_transpose(Δxₙ, batch), J⁻¹, batch), batch)
+        J⁻¹Δfₙ = _batched_mul(J⁻¹, Δfₙ)
+        J⁻¹ += _batched_mul(((Δxₙ .- J⁻¹Δfₙ) ./
+                             (_batched_mul(_batch_transpose(Δxₙ), J⁻¹Δfₙ) .+ T(1e-5))),
+                            _batched_mul(_batch_transpose(Δxₙ), J⁻¹))
 
         iszero(fₙ) &&
             return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
@@ -78,5 +78,4 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
     return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
 end
 
-
-end
\ No newline at end of file
+end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 51785e323..b82a7d0c6 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -16,9 +16,7 @@ end
 
 function __init__()
     @static if !isdefined(Base, :get_extension)
-        @require NNlib="872c559c-99b0-510c-b3b7-b6c96a88d5cd" begin
-            include("../ext/SimpleBatchedNonlinearSolveExt.jl")
-        end
+        @require NNlib="872c559c-99b0-510c-b3b7-b6c96a88d5cd" begin include("../ext/SimpleBatchedNonlinearSolveExt.jl") end
     end
 end
 
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 15da8a77e..d0ae233b3 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -3,6 +3,11 @@
 
 A low-overhead implementation of Broyden. This method is non-allocating on scalar
 and static array problems.
+
+!!! note
+
+    To use the `batched` version, remember to load `NNlib`, i.e., `using NNlib` or
+    `import NNlib` must be present in your code.
 """
 struct Broyden{batched} <: AbstractSimpleNonlinearSolveAlgorithm
     Broyden(; batched = false) = new{batched}()
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 3f386d276..12525d814 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -370,3 +370,15 @@ for options in list_of_options
     sol = solve(probN, alg)
     @test all(abs.(f(u, p)) .< 1e-10)
 end
+
+# Batched Broyden
+using NNlib
+
+f, u0 = (u, p) -> u .* u .- p, randn(1, 3)
+
+p = [2.0 1.0 5.0];
+probN = NonlinearProblem{false}(f, u0, p);
+
+sol = solve(probN, Broyden(batched = true))
+
+@test abs.(sol.u) ≈ sqrt.(p)

From 940fef52dc8618ee56cb0d229f22de9c3ad44d3e Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 16 Feb 2023 17:25:51 -0500
Subject: [PATCH 073/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 28e3410a9..2d99b8849 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.12"
+version = "0.1.13"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"

From 918f72e311e78bebaebc9a48cd8195df896cd9b8 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 16 Feb 2023 17:26:40 -0500
Subject: [PATCH 074/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 2d99b8849..0e05f54ba 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.13"
+version = "0.1.11"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"

From 9c7f959366f5b2f48dd39a23d6b950cd95d3e874 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 16 Feb 2023 17:37:09 -0500
Subject: [PATCH 075/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 0e05f54ba..aac1c5413 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -28,6 +28,7 @@ FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 NNlib = "0.8"
 Reexport = "0.2, 1"
+Requires = "1"
 SciMLBase = "1.73"
 SnoopPrecompile = "1"
 StaticArraysCore = "1.4"

From cf4449795dd7b11c09a1de686a51a6dede708edb Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 17 Feb 2023 17:45:59 -0500
Subject: [PATCH 076/375] Add Termination Conditions to Broyden

---
 lib/SimpleNonlinearSolve/Project.toml         |  6 +-
 .../ext/SimpleBatchedNonlinearSolveExt.jl     | 27 +++---
 lib/SimpleNonlinearSolve/src/broyden.jl       | 41 ++++++---
 lib/SimpleNonlinearSolve/test/basictests.jl   | 90 ++++++++++++-------
 4 files changed, 107 insertions(+), 57 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index aac1c5413..0368644a4 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.11"
+version = "0.1.12"
 
 [deps]
 ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
@@ -23,7 +23,7 @@ SimpleBatchedNonlinearSolveExt = "NNlib"
 
 [compat]
 ArrayInterfaceCore = "0.1.1"
-DiffEqBase = "6.114"
+DiffEqBase = "6.118.1"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 NNlib = "0.8"
@@ -43,4 +43,4 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib"]
+test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib", "DiffEqBase"]
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
index d59970471..76c81efc4 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
@@ -1,6 +1,6 @@
 module SimpleBatchedNonlinearSolveExt
 
-using ArrayInterfaceCore, LinearAlgebra, SimpleNonlinearSolve, SciMLBase
+using ArrayInterfaceCore, DiffEqBase, LinearAlgebra, SimpleNonlinearSolve, SciMLBase
 isdefined(Base, :get_extension) ? (using NNlib) : (using ..NNlib)
 
 _batch_transpose(x) = reshape(x, 1, size(x)...)
@@ -31,6 +31,8 @@ end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
                            abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
+    tc = alg.termination_condition
+    mode = DiffEqBase.get_termination_mode(tc)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
 
@@ -47,8 +49,17 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
     end
 
     atol = abstol !== nothing ? abstol :
-           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
-    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+           (tc.abstol !== nothing ? tc.abstol :
+            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
+    rtol = reltol !== nothing ? reltol :
+           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
+
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        error("Broyden currently doesn't support SAFE_BEST termination modes")
+    end
+
+    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? Dict() : nothing
+    termination_condition = tc(storage)
 
     xₙ = x
     xₙ₋₁ = x
@@ -63,14 +74,10 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
                              (_batched_mul(_batch_transpose(Δxₙ), J⁻¹Δfₙ) .+ T(1e-5))),
                             _batched_mul(_batch_transpose(Δxₙ), J⁻¹))
 
-        iszero(fₙ) &&
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                            retcode = ReturnCode.Success)
-
-        if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                            retcode = ReturnCode.Success)
+        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
         end
+
         xₙ₋₁ = xₙ
         fₙ₋₁ = fₙ
     end
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index d0ae233b3..e8d339c4c 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -1,5 +1,7 @@
 """
-    Broyden(; batched = false)
+    Broyden(; batched = false,
+            termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+                                                                abstol = nothing, reltol = nothing))
 
 A low-overhead implementation of Broyden. This method is non-allocating on scalar
 and static array problems.
@@ -9,12 +11,22 @@ and static array problems.
     To use the `batched` version, remember to load `NNlib`, i.e., `using NNlib` or
     `import NNlib` must be present in your code.
 """
-struct Broyden{batched} <: AbstractSimpleNonlinearSolveAlgorithm
-    Broyden(; batched = false) = new{batched}()
+struct Broyden{batched, TC <: NLSolveTerminationCondition} <:
+       AbstractSimpleNonlinearSolveAlgorithm
+    termination_condition::TC
+
+    function Broyden(; batched = false,
+                     termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+                                                                         abstol = nothing,
+                                                                         reltol = nothing))
+        return new{batched, typeof(termination_condition)}(termination_condition)
+    end
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{false}, args...;
                            abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
+    tc = alg.termination_condition
+    mode = DiffEqBase.get_termination_mode(tc)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
 
@@ -27,8 +39,17 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{false}, args...;
     end
 
     atol = abstol !== nothing ? abstol :
-           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
-    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+           (tc.abstol !== nothing ? tc.abstol :
+            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
+    rtol = reltol !== nothing ? reltol :
+           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
+
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        error("Broyden currently doesn't support SAFE_BEST termination modes")
+    end
+
+    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? Dict() : nothing
+    termination_condition = tc(storage)
 
     xₙ = x
     xₙ₋₁ = x
@@ -41,14 +62,10 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{false}, args...;
         J⁻¹Δfₙ = J⁻¹ * Δfₙ
         J⁻¹ += ((Δxₙ .- J⁻¹Δfₙ) ./ (Δxₙ' * J⁻¹Δfₙ)) * (Δxₙ' * J⁻¹)
 
-        iszero(fₙ) &&
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                            retcode = ReturnCode.Success)
-
-        if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                            retcode = ReturnCode.Success)
+        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
         end
+
         xₙ₋₁ = xₙ
         fₙ₋₁ = fₙ
     end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 12525d814..819621202 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,8 +1,25 @@
 using SimpleNonlinearSolve
 using StaticArrays
 using BenchmarkTools
+using DiffEqBase
 using Test
 
+const BATCHED_BROYDEN_SOLVERS = Broyden[]
+const BROYDEN_SOLVERS = Broyden[]
+
+for mode in instances(NLSolveTerminationMode.T)
+    if mode ∈
+       (NLSolveTerminationMode.SteadyStateDefault, NLSolveTerminationMode.RelSafeBest,
+        NLSolveTerminationMode.AbsSafeBest)
+        continue
+    end
+
+    termination_condition = NLSolveTerminationCondition(mode; abstol = nothing,
+                                                        reltol = nothing)
+    push!(BROYDEN_SOLVERS, Broyden(; batched = false, termination_condition))
+    push!(BATCHED_BROYDEN_SOLVERS, Broyden(; batched = true, termination_condition))
+end
+
 # SimpleNewtonRaphson
 function benchmark_scalar(f, u0)
     probN = NonlinearProblem{false}(f, u0)
@@ -27,16 +44,19 @@ if VERSION >= v"1.7"
 end
 
 # Broyden
-function benchmark_scalar(f, u0)
+function benchmark_scalar(f, u0, alg)
     probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, Broyden()))
+    sol = (solve(probN, alg))
 end
 
-sol = benchmark_scalar(sf, csu0)
-@test sol.retcode === ReturnCode.Success
-@test sol.u * sol.u - 2 < 1e-9
-if VERSION >= v"1.7"
-    @test (@ballocated benchmark_scalar(sf, csu0)) == 0
+for alg in BROYDEN_SOLVERS
+    sol = benchmark_scalar(sf, csu0, alg)
+    @test sol.retcode === ReturnCode.Success
+    @test sol.u * sol.u - 2 < 1e-9
+    # FIXME: Termination Condition Implementation is allocating. Not sure how to fix it.
+    # if VERSION >= v"1.7"
+    #     @test (@ballocated benchmark_scalar($sf, $csu0, $termination_condition)) == 0
+    # end
 end
 
 # Klement
@@ -78,8 +98,8 @@ using ForwardDiff
 # Immutable
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
-for alg in (SimpleNewtonRaphson(), Broyden(), LBroyden(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane())
+for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
+            SimpleDFSane(), BROYDEN_SOLVERS...)
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, abstol = 1e-9)
@@ -94,8 +114,8 @@ end
 
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
-for alg in (SimpleNewtonRaphson(), Broyden(), LBroyden(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane())
+for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
+            SimpleDFSane(), BROYDEN_SOLVERS...)
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -160,8 +180,8 @@ for alg in [Bisection(), Falsi(), Ridder(), Brent()]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
 
-for alg in (SimpleNewtonRaphson(), Broyden(), LBroyden(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane())
+for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
+            SimpleDFSane(), BROYDEN_SOLVERS...)
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -176,14 +196,15 @@ end
 f, u0 = (u, p) -> u .* u .- 2.0, @SVector[1.0, 1.0]
 probN = NonlinearProblem(f, u0)
 
-@test solve(probN, SimpleNewtonRaphson()).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleTrustRegion()).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleTrustRegion(; autodiff = false)).u[end] ≈ sqrt(2.0)
-@test solve(probN, Broyden()).u[end] ≈ sqrt(2.0)
-@test solve(probN, LBroyden()).u[end] ≈ sqrt(2.0)
-@test solve(probN, Klement()).u[end] ≈ sqrt(2.0)
-@test solve(probN, SimpleDFSane()).u[end] ≈ sqrt(2.0)
+for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
+            SimpleTrustRegion(),
+            SimpleTrustRegion(; autodiff = false), LBroyden(), Klement(), SimpleDFSane(),
+            BROYDEN_SOLVERS...)
+    sol = solve(probN, alg)
+
+    @test sol.retcode == ReturnCode.Success
+    @test sol.u[end] ≈ sqrt(2.0)
+end
 
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol
@@ -191,18 +212,16 @@ for u0 in [1.0, [1, 1.0]]
     probN = NonlinearProblem(f, u0)
     sol = sqrt(2) * u0
 
-    @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
-    @test solve(probN, SimpleNewtonRaphson()).u ≈ sol
-    @test solve(probN, SimpleNewtonRaphson(; autodiff = false)).u ≈ sol
-
-    @test solve(probN, SimpleTrustRegion()).u ≈ sol
-    @test solve(probN, SimpleTrustRegion()).u ≈ sol
-    @test solve(probN, SimpleTrustRegion(; autodiff = false)).u ≈ sol
+    for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
+                SimpleTrustRegion(),
+                SimpleTrustRegion(; autodiff = false), LBroyden(), Klement(),
+                SimpleDFSane(),
+                BROYDEN_SOLVERS...)
+        sol2 = solve(probN, alg)
 
-    @test solve(probN, Broyden()).u ≈ sol
-    @test solve(probN, LBroyden()).u ≈ sol
-    @test solve(probN, Klement()).u ≈ sol
-    @test solve(probN, SimpleDFSane()).u ≈ sol
+        @test sol2.retcode == ReturnCode.Success
+        @test sol2.u ≈ sol
+    end
 end
 
 # Bisection Tests
@@ -382,3 +401,10 @@ probN = NonlinearProblem{false}(f, u0, p);
 sol = solve(probN, Broyden(batched = true))
 
 @test abs.(sol.u) ≈ sqrt.(p)
+
+for alg in BATCHED_BROYDEN_SOLVERS
+    sol = solve(probN, alg)
+
+    @test sol.retcode == ReturnCode.Success
+    @test abs.(sol.u) ≈ sqrt.(p)
+end

From 94eaa39e62d6481895b1ba8cded4226be12a6d7a Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 18 Feb 2023 09:21:18 -0500
Subject: [PATCH 077/375] update to ArrayInterface v7

---
 lib/SimpleNonlinearSolve/Project.toml                         | 4 ++--
 .../ext/SimpleBatchedNonlinearSolveExt.jl                     | 4 ++--
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl          | 2 +-
 lib/SimpleNonlinearSolve/src/utils.jl                         | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index aac1c5413..87936fe44 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -4,7 +4,7 @@ authors = ["SciML"]
 version = "0.1.11"
 
 [deps]
-ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2"
+ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
@@ -22,7 +22,7 @@ NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 SimpleBatchedNonlinearSolveExt = "NNlib"
 
 [compat]
-ArrayInterfaceCore = "0.1.1"
+ArrayInterface = "7"
 DiffEqBase = "6.114"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
index d59970471..4d5b45551 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
@@ -1,6 +1,6 @@
 module SimpleBatchedNonlinearSolveExt
 
-using ArrayInterfaceCore, LinearAlgebra, SimpleNonlinearSolve, SciMLBase
+using ArrayInterface, LinearAlgebra, SimpleNonlinearSolve, SciMLBase
 isdefined(Base, :get_extension) ? (using NNlib) : (using ..NNlib)
 
 _batch_transpose(x) = reshape(x, 1, size(x)...)
@@ -20,7 +20,7 @@ function _batched_mul(x::AbstractArray{T1, 3}, y::AbstractArray{T2, 3}) where {T
 end
 
 function _init_J_batched(x::AbstractMatrix{T}) where {T}
-    J = ArrayInterfaceCore.zeromatrix(x[:, 1])
+    J = ArrayInterface.zeromatrix(x[:, 1])
     if ismutable(x)
         J[diagind(J)] .= one(eltype(x))
     else
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index b82a7d0c6..7e6ef3441 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -5,7 +5,7 @@ using FiniteDiff, ForwardDiff
 using ForwardDiff: Dual
 using StaticArraysCore
 using LinearAlgebra
-import ArrayInterfaceCore
+import ArrayInterface
 using DiffEqBase
 
 @reexport using SciMLBase
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index f3b7de9f6..890aa2415 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -35,7 +35,7 @@ value(x::Dual) = ForwardDiff.value(x)
 value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
 
 function init_J(x)
-    J = ArrayInterfaceCore.zeromatrix(x)
+    J = ArrayInterface.zeromatrix(x)
     if ismutable(x)
         J[diagind(J)] .= one(eltype(x))
     else

From 6b9a5b2f6fb9d21a60532eacfbc718894579915a Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 18 Feb 2023 09:47:28 -0500
Subject: [PATCH 078/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 87936fe44..be3a13cba 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.11"
+version = "0.1.12"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

From 4724ed3eb4262c0f8436809e5f1425bcc9c44160 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 18 Feb 2023 23:13:09 -0500
Subject: [PATCH 079/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index be3a13cba..4d78f7d62 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -22,7 +22,7 @@ NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 SimpleBatchedNonlinearSolveExt = "NNlib"
 
 [compat]
-ArrayInterface = "7"
+ArrayInterface = "6, 7"
 DiffEqBase = "6.114"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"

From 6ff49047886de4d2c1de6fa33351bec5d086ad9f Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 19 Feb 2023 00:29:16 +0100
Subject: [PATCH 080/375] Added halley's method

---
 lib/SimpleNonlinearSolve/src/halley.jl | 90 ++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 lib/SimpleNonlinearSolve/src/halley.jl

diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
new file mode 100644
index 000000000..f428db336
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -0,0 +1,90 @@
+"""
+```julia
+Halley(; chunk_size = Val{0}(), autodiff = Val{true}(),
+                                 diff_type = Val{:forward})
+```
+
+A low-overhead implementation of Halley's Method. This method is non-allocating on scalar
+and static array problems.
+
+!!! note
+
+    As part of the decreased overhead, this method omits some of the higher level error
+    catching of the other methods. Thus, to see better error messages, use one of the other
+    methods like `NewtonRaphson`
+
+### Keyword Arguments
+
+- `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
+  system. This allows for multiple derivative columns to be computed simultaneously,
+  improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
+  default chunk size mechanism. For more details, see the documentation for
+  [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
+- `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
+  Note that this argument is ignored if an analytical Jacobian is passed; as that will be
+  used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
+  If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
+- `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
+  `Val{:forward}` for forward finite differences. For more details on the choices, see the
+  [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+"""
+struct Halley{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
+    function Halley(; chunk_size = Val{0}(), autodiff = Val{true}(),
+                                 diff_type = Val{:forward})
+        new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
+            SciMLBase._unwrap_val(diff_type)}()
+    end
+end
+
+function SciMLBase.__solve(prob::NonlinearProblem,
+                           alg::Halley, args...; abstol = nothing,
+                           reltol = nothing,
+                           maxiters = 1000, kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    x = float(prob.u0)
+    fx = f(x)
+    # fx = float(prob.u0)
+    if !isa(fx, Number) || !isa(x, Number)
+        error("Halley currently only supports scalar-valued single-variable functions")
+    end
+    T = typeof(x)
+
+    if SciMLBase.isinplace(prob)
+        error("Halley currently only supports out-of-place nonlinear problems")
+    end
+
+    atol = abstol !== nothing ? abstol :
+           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
+    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+
+    if typeof(x) <: Number
+        xo = oftype(one(eltype(x)), Inf)
+    else
+        xo = map(x -> oftype(one(eltype(x)), Inf), x)
+    end
+
+    for i in 1:maxiters
+        if alg_autodiff(alg)
+            fx = f(x)
+            dfdx(x) = ForwardDiff.derivative(f, x)
+            dfx = dfdx(x)
+            d2fx = ForwardDiff.derivative(dfdx, x)
+        else
+            fx = f(x)
+            dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x),
+                                                          fx)
+            d2fx = FiniteDiff.finite_difference_derivative(x -> FiniteDiff.finite_difference_derivative(f, x),
+                                                           x, diff_type(alg), eltype(x), fx)
+        end
+        iszero(fx) &&
+            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+        Δx = (2*dfx^2 - fx*d2fx) \ 2fx*dfx
+        x -= Δx
+        if isapprox(x, xo, atol = atol, rtol = rtol)
+            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+        end
+        xo = x
+    end
+
+    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end

From becd1e6660cc3bde0c6197aeb70062ce4caf51b2 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 19 Feb 2023 00:34:00 +0100
Subject: [PATCH 081/375] added some tests for halley

---
 .../src/SimpleNonlinearSolve.jl               |  7 ++--
 lib/SimpleNonlinearSolve/test/basictests.jl   | 34 +++++++++++++++++--
 2 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 7e6ef3441..dfc825c52 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -5,7 +5,7 @@ using FiniteDiff, ForwardDiff
 using ForwardDiff: Dual
 using StaticArraysCore
 using LinearAlgebra
-import ArrayInterface
+import ArrayInterfaceCore
 using DiffEqBase
 
 @reexport using SciMLBase
@@ -37,12 +37,13 @@ include("ridder.jl")
 include("brent.jl")
 include("dfsane.jl")
 include("ad.jl")
+include("halley.jl")
 
 import SnoopPrecompile
 
 SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-    for alg in (SimpleNewtonRaphson, Broyden, Klement, SimpleTrustRegion, SimpleDFSane)
+    for alg in (SimpleNewtonRaphson, Halley, Broyden, Klement, SimpleTrustRegion, SimpleDFSane)
         solve(prob_no_brack, alg(), abstol = T(1e-2))
     end
 
@@ -63,7 +64,7 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
 end end
 
 # DiffEq styled algorithms
-export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Klement,
+export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
        Ridder, SimpleNewtonRaphson, SimpleTrustRegion
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 12525d814..199e55579 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -26,6 +26,29 @@ if VERSION >= v"1.7"
     @test (@ballocated benchmark_scalar(sf, csu0)) == 0
 end
 
+# Halley
+function benchmark_scalar(f, u0)
+    probN = NonlinearProblem{false}(f, u0)
+    sol = (solve(probN, Halley()))
+end
+
+# function ff(u, p)
+#     u .* u .- 2
+# end
+# const cu0 = @SVector[1.0, 1.0]
+function sf(u, p)
+    u * u - 2
+end
+const csu0 = 1.0
+
+sol = benchmark_scalar(sf, csu0)
+@test sol.retcode === ReturnCode.Success
+@test sol.u * sol.u - 2 < 1e-9
+
+if VERSION >= v"1.7"
+    @test (@ballocated benchmark_scalar(sf, csu0)) == 0
+end
+
 # Broyden
 function benchmark_scalar(f, u0)
     probN = NonlinearProblem{false}(f, u0)
@@ -95,7 +118,7 @@ end
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
 for alg in (SimpleNewtonRaphson(), Broyden(), LBroyden(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane())
+            SimpleDFSane(), Halley())
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -161,7 +184,7 @@ for alg in [Bisection(), Falsi(), Ridder(), Brent()]
 end
 
 for alg in (SimpleNewtonRaphson(), Broyden(), LBroyden(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane())
+            SimpleDFSane(), Halley())
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -185,6 +208,13 @@ probN = NonlinearProblem(f, u0)
 @test solve(probN, Klement()).u[end] ≈ sqrt(2.0)
 @test solve(probN, SimpleDFSane()).u[end] ≈ sqrt(2.0)
 
+# Separate Error check for Halley; will be included in above error checks for the improved Halley
+f, u0 = (u, p) -> u * u - 2.0, 1.0
+probN = NonlinearProblem(f, u0)
+
+@test solve(probN, Halley()).u ≈ sqrt(2.0)
+
+
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol
     f = (u, p) -> u .* u .- 2.0

From b1c5f63e0fed3241ade13862c3a2a7728e9abfb2 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Sun, 19 Feb 2023 00:52:32 +0100
Subject: [PATCH 082/375] added tests

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index dfc825c52..e80c7cff2 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -5,7 +5,7 @@ using FiniteDiff, ForwardDiff
 using ForwardDiff: Dual
 using StaticArraysCore
 using LinearAlgebra
-import ArrayInterfaceCore
+import ArrayInterface
 using DiffEqBase
 
 @reexport using SciMLBase

From b07f0d7bc3709fca5203764569cdd77e187a8403 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 18 Feb 2023 23:10:37 -0500
Subject: [PATCH 083/375] Update src/halley.jl

---
 lib/SimpleNonlinearSolve/src/halley.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index f428db336..5c85afe20 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -78,7 +78,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         end
         iszero(fx) &&
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-        Δx = (2*dfx^2 - fx*d2fx) \ 2fx*dfx
+        Δx = (2*dfx^2 - fx*d2fx) \ (2fx*dfx)
         x -= Δx
         if isapprox(x, xo, atol = atol, rtol = rtol)
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)

From 6cdd257d63335e6162f999cd489317aaa853f24e Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Sun, 19 Feb 2023 07:59:38 -0500
Subject: [PATCH 084/375] format

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 3 ++-
 lib/SimpleNonlinearSolve/src/halley.jl               | 7 ++++---
 lib/SimpleNonlinearSolve/test/basictests.jl          | 1 -
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index e80c7cff2..4c90b094f 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -43,7 +43,8 @@ import SnoopPrecompile
 
 SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-    for alg in (SimpleNewtonRaphson, Halley, Broyden, Klement, SimpleTrustRegion, SimpleDFSane)
+    for alg in (SimpleNewtonRaphson, Halley, Broyden, Klement, SimpleTrustRegion,
+                SimpleDFSane)
         solve(prob_no_brack, alg(), abstol = T(1e-2))
     end
 
diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index 5c85afe20..77ea3a4a1 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -30,7 +30,7 @@ and static array problems.
 """
 struct Halley{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     function Halley(; chunk_size = Val{0}(), autodiff = Val{true}(),
-                                 diff_type = Val{:forward})
+                    diff_type = Val{:forward})
         new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
             SciMLBase._unwrap_val(diff_type)}()
     end
@@ -73,12 +73,13 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             fx = f(x)
             dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x),
                                                           fx)
-            d2fx = FiniteDiff.finite_difference_derivative(x -> FiniteDiff.finite_difference_derivative(f, x),
+            d2fx = FiniteDiff.finite_difference_derivative(x -> FiniteDiff.finite_difference_derivative(f,
+                                                                                                        x),
                                                            x, diff_type(alg), eltype(x), fx)
         end
         iszero(fx) &&
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-        Δx = (2*dfx^2 - fx*d2fx) \ (2fx*dfx)
+        Δx = (2 * dfx^2 - fx * d2fx) \ (2fx * dfx)
         x -= Δx
         if isapprox(x, xo, atol = atol, rtol = rtol)
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 199e55579..5f8f17a89 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -214,7 +214,6 @@ probN = NonlinearProblem(f, u0)
 
 @test solve(probN, Halley()).u ≈ sqrt(2.0)
 
-
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol
     f = (u, p) -> u .* u .- 2.0

From 427fae30024c9fbe62228edeea3caebe3cc561ee Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sun, 19 Feb 2023 10:01:24 -0500
Subject: [PATCH 085/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 9ad93b17c..796e5416b 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -23,7 +23,6 @@ SimpleBatchedNonlinearSolveExt = "NNlib"
 
 [compat]
 ArrayInterface = "6, 7"
-ArrayInterfaceCore = "0.1.1"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 NNlib = "0.8"

From e0ff9fc09d3a61d60d0de56ee8ec0adf35555092 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sun, 19 Feb 2023 10:10:37 -0500
Subject: [PATCH 086/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 796e5416b..78f3fcc88 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -42,4 +42,4 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib", "DiffEqBase"]
+test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib"]

From 2a3c9e1f7c4e81d1d6d9baa77bc4e044d60d2c7a Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sun, 19 Feb 2023 10:33:18 -0500
Subject: [PATCH 087/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 78f3fcc88..8f52dbdc6 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.12"
+version = "0.1.13"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

From 9b04692b2e84006cfef1cafe9ad5c8e319d83f4a Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sun, 19 Feb 2023 12:06:11 -0500
Subject: [PATCH 088/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 8f52dbdc6..92e852cb9 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -23,6 +23,7 @@ SimpleBatchedNonlinearSolveExt = "NNlib"
 
 [compat]
 ArrayInterface = "6, 7"
+DiffEqBase = "6.119"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 NNlib = "0.8"

From 3ec0acc1e2c3d68248727b9097e11c85d56599b8 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Mon, 20 Feb 2023 23:47:57 +0100
Subject: [PATCH 089/375] multivariate halley and some tests

---
 lib/SimpleNonlinearSolve/src/halley.jl      | 77 ++++++++++++++++-----
 lib/SimpleNonlinearSolve/test/basictests.jl | 21 +++---
 2 files changed, 71 insertions(+), 27 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index 77ea3a4a1..557f43473 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -42,10 +42,28 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                            maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
-    fx = f(x)
-    # fx = float(prob.u0)
-    if !isa(fx, Number) || !isa(x, Number)
-        error("Halley currently only supports scalar-valued single-variable functions")
+    # Defining all derivative expressions in one place before the iterations
+    if isa(x, AbstractArray)
+        if alg_autodiff(alg)
+            n = length(x)
+            a_dfdx(x) = ForwardDiff.jacobian(f, x)
+            a_d2fdx(x) = ForwardDiff.jacobian(a_dfdx, x)
+            A = Array{Union{Nothing, Number}}(nothing, n, n)
+            #fx = f(x)
+        else
+            n = length(x)
+            f_dfdx(x) = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x))
+            f_d2fdx(x) = FiniteDiff.finite_difference_jacobian(f_dfdx, x, diff_type(alg), eltype(x))
+            A = Array{Union{Nothing, Number}}(nothing, n, n)
+        end
+    elseif isa(x, Number)
+        if alg_autodiff(alg)
+            sa_dfdx(x) = ForwardDiff.derivative(f, x)
+            sa_d2fdx(x) = ForwardDiff.derivative(sa_dfdx, x)
+        else
+            sf_dfdx(x) = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x))
+            sf_d2fdx(x) = FiniteDiff.finite_difference_derivative(sf_dfdx, x, diff_type(alg), eltype(x))
+        end
     end
     T = typeof(x)
 
@@ -65,22 +83,49 @@ function SciMLBase.__solve(prob::NonlinearProblem,
 
     for i in 1:maxiters
         if alg_autodiff(alg)
-            fx = f(x)
-            dfdx(x) = ForwardDiff.derivative(f, x)
-            dfx = dfdx(x)
-            d2fx = ForwardDiff.derivative(dfdx, x)
+            if isa(x, Number)
+                fx = f(x)
+                dfx = sa_dfdx(x)
+                d2fx = sa_d2fdx(x)
+            else
+                fx = f(x)
+                dfx = a_dfdx(x)
+                d2fx = reshape(a_d2fdx(x), (n,n,n)) # A 3-dim Hessian Tensor
+                ai = -(dfx \ fx)
+                for j in 1:n
+                    tmp = transpose(d2fx[:, :, j] * ai)
+                    A[j, :] = tmp
+                end
+                bi = (dfx) \ (A * ai)
+                ci = (ai .* ai) ./ (ai .+ (0.5 .* bi))
+            end
         else
-            fx = f(x)
-            dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x),
-                                                          fx)
-            d2fx = FiniteDiff.finite_difference_derivative(x -> FiniteDiff.finite_difference_derivative(f,
-                                                                                                        x),
-                                                           x, diff_type(alg), eltype(x), fx)
+            if isa(x, Number)
+                fx = f(x)
+                dfx = sf_dfdx(x)
+                d2fx = sf_d2fdx(x)
+            else
+                fx = f(x)
+                dfx = f_dfdx(x)
+                d2fx = reshape(f_d2fdx(x), (n,n,n)) # A 3-dim Hessian Tensor
+                ai = -(dfx \ fx)
+                for j in 1:n
+                    tmp = transpose(d2fx[:, :, j] * ai)
+                    A[j, :] = tmp
+                end
+                bi = (dfx) \ (A * ai)
+                ci = (ai .* ai) ./ (ai .+ (0.5 .* bi))
+            end
         end
         iszero(fx) &&
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-        Δx = (2 * dfx^2 - fx * d2fx) \ (2fx * dfx)
-        x -= Δx
+        if isa(x, Number)
+            Δx = (2 * dfx^2 - fx * d2fx) \ (2fx * dfx)
+            x -= Δx
+        else
+            Δx = ci
+            x += Δx
+        end
         if isapprox(x, xo, atol = atol, rtol = rtol)
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
         end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 26e92dfe8..8fbdfffaf 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -49,10 +49,10 @@ function benchmark_scalar(f, u0)
     sol = (solve(probN, Halley()))
 end
 
-# function ff(u, p)
-#     u .* u .- 2
-# end
-# const cu0 = @SVector[1.0, 1.0]
+function ff(u, p)
+    u .* u .- 2
+end
+const cu0 = @SVector[1.0, 1.0]
 function sf(u, p)
     u * u - 2
 end
@@ -62,6 +62,10 @@ sol = benchmark_scalar(sf, csu0)
 @test sol.retcode === ReturnCode.Success
 @test sol.u * sol.u - 2 < 1e-9
 
+sol = benchmark_scalar(ff, cu0)
+@test sol.retcode === ReturnCode.Success
+@test sol.u .* sol.u .- 2 < [1e-9, 1e-9]
+
 if VERSION >= v"1.7"
     @test (@ballocated benchmark_scalar(sf, csu0)) == 0
 end
@@ -122,7 +126,7 @@ using ForwardDiff
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
 for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane(), BROYDEN_SOLVERS...)
+            SimpleDFSane(), Halley(), BROYDEN_SOLVERS...)
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, abstol = 1e-9)
@@ -221,7 +225,7 @@ probN = NonlinearProblem(f, u0)
 
 for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
             SimpleTrustRegion(),
-            SimpleTrustRegion(; autodiff = false), LBroyden(), Klement(), SimpleDFSane(),
+            SimpleTrustRegion(; autodiff = false), Halley(), Halley(; autodiff = false), LBroyden(), Klement(), SimpleDFSane(),
             BROYDEN_SOLVERS...)
     sol = solve(probN, alg)
 
@@ -229,11 +233,6 @@ for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
     @test sol.u[end] ≈ sqrt(2.0)
 end
 
-# Separate Error check for Halley; will be included in above error checks for the improved Halley
-f, u0 = (u, p) -> u * u - 2.0, 1.0
-probN = NonlinearProblem(f, u0)
-
-@test solve(probN, Halley()).u ≈ sqrt(2.0)
 
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol

From 5f531f7f1e923dd68dbd009142d89116f0c7bb67 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 21 Feb 2023 22:42:23 +0100
Subject: [PATCH 090/375] modified allocations

---
 lib/SimpleNonlinearSolve/src/halley.jl | 50 +++++++-------------------
 1 file changed, 13 insertions(+), 37 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index 557f43473..70b8a7f8c 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -42,28 +42,8 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                            maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
-    # Defining all derivative expressions in one place before the iterations
     if isa(x, AbstractArray)
-        if alg_autodiff(alg)
-            n = length(x)
-            a_dfdx(x) = ForwardDiff.jacobian(f, x)
-            a_d2fdx(x) = ForwardDiff.jacobian(a_dfdx, x)
-            A = Array{Union{Nothing, Number}}(nothing, n, n)
-            #fx = f(x)
-        else
-            n = length(x)
-            f_dfdx(x) = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x))
-            f_d2fdx(x) = FiniteDiff.finite_difference_jacobian(f_dfdx, x, diff_type(alg), eltype(x))
-            A = Array{Union{Nothing, Number}}(nothing, n, n)
-        end
-    elseif isa(x, Number)
-        if alg_autodiff(alg)
-            sa_dfdx(x) = ForwardDiff.derivative(f, x)
-            sa_d2fdx(x) = ForwardDiff.derivative(sa_dfdx, x)
-        else
-            sf_dfdx(x) = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x))
-            sf_d2fdx(x) = FiniteDiff.finite_difference_derivative(sf_dfdx, x, diff_type(alg), eltype(x))
-        end
+        n = length(x)
     end
     T = typeof(x)
 
@@ -85,34 +65,30 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         if alg_autodiff(alg)
             if isa(x, Number)
                 fx = f(x)
-                dfx = sa_dfdx(x)
-                d2fx = sa_d2fdx(x)
+                dfx = ForwardDiff.derivative(f, x)
+                d2fx = ForwardDiff.derivative(x -> ForwardDiff.derivative(f, x), x)
             else
                 fx = f(x)
-                dfx = a_dfdx(x)
-                d2fx = reshape(a_d2fdx(x), (n,n,n)) # A 3-dim Hessian Tensor
+                dfx = ForwardDiff.jacobian(f, x)
+                d2fx = ForwardDiff.jacobian(x -> ForwardDiff.jacobian(f, x), x) #  n^2 by n matrix
                 ai = -(dfx \ fx)
-                for j in 1:n
-                    tmp = transpose(d2fx[:, :, j] * ai)
-                    A[j, :] = tmp
-                end
+                A = reshape(d2fx * ai, (n, n))
                 bi = (dfx) \ (A * ai)
                 ci = (ai .* ai) ./ (ai .+ (0.5 .* bi))
             end
         else
             if isa(x, Number)
                 fx = f(x)
-                dfx = sf_dfdx(x)
-                d2fx = sf_d2fdx(x)
+                dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x))
+                d2fx = FiniteDiff.finite_difference_derivative(x -> FiniteDiff.finite_difference_derivative(f, x), x,
+                                                                                            diff_type(alg), eltype(x))
             else
                 fx = f(x)
-                dfx = f_dfdx(x)
-                d2fx = reshape(f_d2fdx(x), (n,n,n)) # A 3-dim Hessian Tensor
+                dfx = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x))
+                d2fx = FiniteDiff.finite_difference_jacobian(x -> FiniteDiff.finite_difference_jacobian(f, x), x, 
+                                                                        diff_type(alg), eltype(x))
                 ai = -(dfx \ fx)
-                for j in 1:n
-                    tmp = transpose(d2fx[:, :, j] * ai)
-                    A[j, :] = tmp
-                end
+                A = reshape(d2fx * ai, (n, n))
                 bi = (dfx) \ (A * ai)
                 ci = (ai .* ai) ./ (ai .+ (0.5 .* bi))
             end

From 7f542660bbeeb2db088f70db93a90c6f970d8a64 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 23 Feb 2023 01:29:26 +0100
Subject: [PATCH 091/375] fixed allocations and tests pass

---
 lib/SimpleNonlinearSolve/src/halley.jl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index 70b8a7f8c..0e5417642 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -42,6 +42,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                            maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
+    fx = f(x)
     if isa(x, AbstractArray)
         n = length(x)
     end
@@ -70,7 +71,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             else
                 fx = f(x)
                 dfx = ForwardDiff.jacobian(f, x)
-                d2fx = ForwardDiff.jacobian(x -> ForwardDiff.jacobian(f, x), x) #  n^2 by n matrix
+                d2fx = ForwardDiff.jacobian(x -> ForwardDiff.jacobian(f, x), x)
                 ai = -(dfx \ fx)
                 A = reshape(d2fx * ai, (n, n))
                 bi = (dfx) \ (A * ai)

From 0882d7629ef0e32ec0bd7a34f1015159272192ca Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 8 Mar 2023 12:39:57 -0500
Subject: [PATCH 092/375] Add batched lbroyden

---
 lib/SimpleNonlinearSolve/Project.toml       |   2 +-
 lib/SimpleNonlinearSolve/src/halley.jl      |  15 ++-
 lib/SimpleNonlinearSolve/src/lbroyden.jl    | 101 +++++++++++++++-----
 lib/SimpleNonlinearSolve/test/basictests.jl |  48 ++++++----
 4 files changed, 121 insertions(+), 45 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 92e852cb9..55ccee5c4 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.13"
+version = "0.1.14"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index 0e5417642..cdda7beda 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -80,14 +80,19 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         else
             if isa(x, Number)
                 fx = f(x)
-                dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x))
-                d2fx = FiniteDiff.finite_difference_derivative(x -> FiniteDiff.finite_difference_derivative(f, x), x,
-                                                                                            diff_type(alg), eltype(x))
+                dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg),
+                                                              eltype(x))
+                d2fx = FiniteDiff.finite_difference_derivative(x -> FiniteDiff.finite_difference_derivative(f,
+                                                                                                            x),
+                                                               x,
+                                                               diff_type(alg), eltype(x))
             else
                 fx = f(x)
                 dfx = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x))
-                d2fx = FiniteDiff.finite_difference_jacobian(x -> FiniteDiff.finite_difference_jacobian(f, x), x, 
-                                                                        diff_type(alg), eltype(x))
+                d2fx = FiniteDiff.finite_difference_jacobian(x -> FiniteDiff.finite_difference_jacobian(f,
+                                                                                                        x),
+                                                             x,
+                                                             diff_type(alg), eltype(x))
                 ai = -(dfx \ fx)
                 A = reshape(d2fx * ai, (n, n))
                 bi = (dfx) \ (A * ai)
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
index f8ed9f5d3..6a3fc0947 100644
--- a/lib/SimpleNonlinearSolve/src/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -1,19 +1,40 @@
 """
-    LBroyden(threshold::Int = 27)
+    LBroyden(; batched = false,
+              termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+                                                                  abstol = nothing, reltol = nothing),
+              threshold::Int = 27)
 
 A limited memory implementation of Broyden. This method applies the L-BFGS scheme to
 Broyden's method.
+
+!!! warn
+
+    This method is not very stable and can diverge even for very simple problems. This has mostly been
+    tested for neural networks in DeepEquilibriumNetworks.jl.
 """
-Base.@kwdef struct LBroyden <: AbstractSimpleNonlinearSolveAlgorithm
-    threshold::Int = 27
+struct LBroyden{batched, TC <: NLSolveTerminationCondition} <:
+       AbstractSimpleNonlinearSolveAlgorithm
+    termination_condition::TC
+    threshold::Int
+
+    function LBroyden(; batched = false, threshold::Int = 27,
+                      termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+                                                                          abstol = nothing,
+                                                                          reltol = nothing))
+        return new{batched, typeof(termination_condition)}(termination_condition, threshold)
+    end
 end
 
-@views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden, args...;
+@views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden{batched}, args...;
                                   abstol = nothing, reltol = nothing, maxiters = 1000,
-                                  batch = false, kwargs...)
+                                  kwargs...) where {batched}
+    tc = alg.termination_condition
+    mode = DiffEqBase.get_termination_mode(tc)
     threshold = min(maxiters, alg.threshold)
     x = float(prob.u0)
 
+    batched && @assert ndims(x)==2 "Batched LBroyden only supports 2D arrays"
+
     if x isa Number
         restore_scalar = true
         x = [x]
@@ -30,12 +51,20 @@ end
         error("LBroyden currently only supports out-of-place nonlinear problems")
     end
 
-    U = fill!(similar(x, (threshold, length(x))), zero(T))
-    Vᵀ = fill!(similar(x, (length(x), threshold)), zero(T))
+    U, Vᵀ = _init_lbroyden_state(batched, x, threshold)
 
     atol = abstol !== nothing ? abstol :
-           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
-    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+           (tc.abstol !== nothing ? tc.abstol :
+            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
+    rtol = reltol !== nothing ? reltol :
+           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
+
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        error("LBroyden currently doesn't support SAFE_BEST termination modes")
+    end
+
+    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? Dict() : nothing
+    termination_condition = tc(storage)
 
     xₙ = x
     xₙ₋₁ = x
@@ -47,27 +76,23 @@ end
         Δxₙ = xₙ .- xₙ₋₁
         Δfₙ = fₙ .- fₙ₋₁
 
-        if iszero(fₙ)
-            xₙ = restore_scalar ? xₙ[] : xₙ
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
-        end
-
-        if isapprox(xₙ, xₙ₋₁; atol, rtol)
+        if termination_condition(restore_scalar ? [fₙ] : fₙ, xₙ, xₙ₋₁, atol, rtol)
             xₙ = restore_scalar ? xₙ[] : xₙ
             return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
         end
 
-        _U = U[1:min(threshold, i), :]
-        _Vᵀ = Vᵀ[:, 1:min(threshold, i)]
+        _U = selectdim(U, 1, 1:min(threshold, i))
+        _Vᵀ = selectdim(Vᵀ, 2, 1:min(threshold, i))
 
         vᵀ = _rmatvec(_U, _Vᵀ, Δxₙ)
         mvec = _matvec(_U, _Vᵀ, Δfₙ)
-        Δxₙ = (Δxₙ .- mvec) ./ (sum(vᵀ .* Δfₙ) .+ convert(T, 1e-5))
+        u = (Δxₙ .- mvec) ./ (sum(vᵀ .* Δfₙ) .+ convert(T, 1e-5))
 
-        Vᵀ[:, mod1(i, threshold)] .= vᵀ
-        U[mod1(i, threshold), :] .= Δxₙ
+        selectdim(Vᵀ, 2, mod1(i, threshold)) .= vᵀ
+        selectdim(U, 1, mod1(i, threshold)) .= u
 
-        update = -_matvec(U[1:min(threshold, i + 1), :], Vᵀ[:, 1:min(threshold, i + 1)], fₙ)
+        update = -_matvec(selectdim(U, 1, 1:min(threshold, i + 1)),
+                          selectdim(Vᵀ, 2, 1:min(threshold, i + 1)), fₙ)
 
         xₙ₋₁ = xₙ
         fₙ₋₁ = fₙ
@@ -77,12 +102,42 @@ end
     return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
 end
 
+function _init_lbroyden_state(batched::Bool, x, threshold)
+    T = eltype(x)
+    if batched
+        U = fill!(similar(x, (threshold, size(x, 1), size(x, 2))), zero(T))
+        Vᵀ = fill!(similar(x, (size(x, 1), threshold, size(x, 2))), zero(T))
+    else
+        U = fill!(similar(x, (threshold, length(x))), zero(T))
+        Vᵀ = fill!(similar(x, (length(x), threshold)), zero(T))
+    end
+    return U, Vᵀ
+end
+
 function _rmatvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
                   x::Union{<:AbstractVector, <:Number})
-    return -x .+ dropdims(sum(U .* sum(Vᵀ .* x; dims = 1)'; dims = 1); dims = 1)
+    length(U) == 0 && return x
+    return -x .+ vec((x' * Vᵀ) * U)
+end
+
+function _rmatvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
+                  x::AbstractMatrix) where {T1, T2}
+    length(U) == 0 && return x
+    Vᵀx = sum(Vᵀ .* reshape(x, size(x, 1), 1, size(x, 2)); dims = 1)
+    return -x .+ _drdims_sum(U .* permutedims(Vᵀx, (2, 1, 3)); dims = 1)
 end
 
 function _matvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
                  x::Union{<:AbstractVector, <:Number})
-    return -x .+ dropdims(sum(sum(x .* U'; dims = 1) .* Vᵀ; dims = 2); dims = 2)
+    length(U) == 0 && return x
+    return -x .+ vec(Vᵀ * (U * x))
 end
+
+function _matvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
+                 x::AbstractMatrix) where {T1, T2}
+    length(U) == 0 && return x
+    xUᵀ = sum(reshape(x, size(x, 1), 1, size(x, 2)) .* permutedims(U, (2, 1, 3)); dims = 1)
+    return -x .+ _drdims_sum(xUᵀ .* Vᵀ; dims = 2)
+end
+
+_drdims_sum(args...; dims = :) = dropdims(sum(args...; dims); dims)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 8fbdfffaf..6906b6f2d 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -6,6 +6,8 @@ using Test
 
 const BATCHED_BROYDEN_SOLVERS = Broyden[]
 const BROYDEN_SOLVERS = Broyden[]
+const BATCHED_LBROYDEN_SOLVERS = LBroyden[]
+const LBROYDEN_SOLVERS = LBroyden[]
 
 for mode in instances(NLSolveTerminationMode.T)
     if mode ∈
@@ -18,6 +20,8 @@ for mode in instances(NLSolveTerminationMode.T)
                                                         reltol = nothing)
     push!(BROYDEN_SOLVERS, Broyden(; batched = false, termination_condition))
     push!(BATCHED_BROYDEN_SOLVERS, Broyden(; batched = true, termination_condition))
+    push!(LBROYDEN_SOLVERS, LBroyden(; batched = false, termination_condition))
+    push!(BATCHED_LBROYDEN_SOLVERS, LBroyden(; batched = true, termination_condition))
 end
 
 # SimpleNewtonRaphson
@@ -134,15 +138,22 @@ for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
     end
 
     for p in 1.1:0.1:100.0
-        @test abs.(g(p)) ≈ sqrt(p)
-        @test abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
+        res = abs.(g(p))
+        # Not surprising if LBrouden fails to converge
+        if any(x -> isnan(x) || x <= 1e-5 || x >= 1e5, res) && alg isa LBroyden
+            @test_broken res ≈ sqrt(p)
+            @test_broken abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
+        else
+            @test res ≈ sqrt(p)
+            @test abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
+        end
     end
 end
 
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
-for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane(), Halley(), BROYDEN_SOLVERS...)
+for alg in (SimpleNewtonRaphson(), Klement(), SimpleTrustRegion(),
+            SimpleDFSane(), Halley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -150,8 +161,15 @@ for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
     end
 
     for p in 1.1:0.1:100.0
-        @test abs(g(p)) ≈ sqrt(p)
-        @test abs(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
+        res = abs.(g(p))
+        # Not surprising if LBrouden fails to converge
+        if any(x -> isnan(x) || x <= 1e-5 || x >= 1e5, res) && alg isa LBroyden
+            @test_broken res ≈ sqrt(p)
+            @test_broken abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
+        else
+            @test res ≈ sqrt(p)
+            @test abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
+        end
     end
 end
 
@@ -207,8 +225,8 @@ for alg in [Bisection(), Falsi(), Ridder(), Brent()]
     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
 end
 
-for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane(), Halley(), BROYDEN_SOLVERS...)
+for alg in (SimpleNewtonRaphson(), Klement(), SimpleTrustRegion(),
+            SimpleDFSane(), Halley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -225,15 +243,15 @@ probN = NonlinearProblem(f, u0)
 
 for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
             SimpleTrustRegion(),
-            SimpleTrustRegion(; autodiff = false), Halley(), Halley(; autodiff = false), LBroyden(), Klement(), SimpleDFSane(),
-            BROYDEN_SOLVERS...)
+            SimpleTrustRegion(; autodiff = false), Halley(), Halley(; autodiff = false),
+            Klement(), SimpleDFSane(),
+            BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
     sol = solve(probN, alg)
 
     @test sol.retcode == ReturnCode.Success
     @test sol.u[end] ≈ sqrt(2.0)
 end
 
-
 for u0 in [1.0, [1, 1.0]]
     local f, probN, sol
     f = (u, p) -> u .* u .- 2.0
@@ -241,10 +259,8 @@ for u0 in [1.0, [1, 1.0]]
     sol = sqrt(2) * u0
 
     for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
-                SimpleTrustRegion(),
-                SimpleTrustRegion(; autodiff = false), LBroyden(), Klement(),
-                SimpleDFSane(),
-                BROYDEN_SOLVERS...)
+                SimpleTrustRegion(), SimpleTrustRegion(; autodiff = false), Klement(),
+                SimpleDFSane(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
         sol2 = solve(probN, alg)
 
         @test sol2.retcode == ReturnCode.Success
@@ -430,7 +446,7 @@ sol = solve(probN, Broyden(batched = true))
 
 @test abs.(sol.u) ≈ sqrt.(p)
 
-for alg in BATCHED_BROYDEN_SOLVERS
+for alg in (BATCHED_BROYDEN_SOLVERS..., BATCHED_LBROYDEN_SOLVERS...)
     sol = solve(probN, alg)
 
     @test sol.retcode == ReturnCode.Success

From 810634d6f1067b213e1fc464efaaa5fea71986b8 Mon Sep 17 00:00:00 2001
From: Hendrik Ranocha <mail@ranocha.de>
Date: Mon, 13 Mar 2023 13:43:27 +0100
Subject: [PATCH 093/375] enable dependabot for GitHub actions

---
 lib/SimpleNonlinearSolve/.github/dependabot.yml | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 lib/SimpleNonlinearSolve/.github/dependabot.yml

diff --git a/lib/SimpleNonlinearSolve/.github/dependabot.yml b/lib/SimpleNonlinearSolve/.github/dependabot.yml
new file mode 100644
index 000000000..700707ced
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/.github/dependabot.yml
@@ -0,0 +1,7 @@
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

From 03fb55dcf78274c8cd1cb7c8dfd1ae08da14d4d8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Mar 2023 13:03:07 +0000
Subject: [PATCH 094/375] Bump actions/checkout from 1 to 3

Bumps [actions/checkout](https://github.com/actions/checkout) from 1 to 3.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v1...v3)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 lib/SimpleNonlinearSolve/.github/workflows/CI.yml          | 2 +-
 lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml  | 4 ++--
 lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index 80a2aea7b..9888c0d5a 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -17,7 +17,7 @@ jobs:
           - '1'
           - '1.6'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
index 6abfcfddc..34f258354 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
@@ -25,14 +25,14 @@ jobs:
           - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceIV}
           - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceV}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.julia-version }}
           arch: x64
       - uses: julia-actions/julia-buildpkg@latest
       - name: Clone Downstream
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ matrix.package.user }}/${{ matrix.package.repo }}
           path: downstream
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
index e4e3512e2..521b8c2b2 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
@@ -21,7 +21,7 @@ jobs:
         with:
           version: ${{ matrix.julia-version }}
 
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
       - name: Install JuliaFormatter and format
         # This will use the latest version by default but you can set the version like so:
         #

From 442b3055a4f397e18e9ac80b1ad6f8822f307cab Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Mar 2023 13:03:13 +0000
Subject: [PATCH 095/375] Bump actions/cache from 1 to 3

Bumps [actions/cache](https://github.com/actions/cache) from 1 to 3.
- [Release notes](https://github.com/actions/cache/releases)
- [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md)
- [Commits](https://github.com/actions/cache/compare/v1...v3)

---
updated-dependencies:
- dependency-name: actions/cache
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 lib/SimpleNonlinearSolve/.github/workflows/CI.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index 80a2aea7b..5eca1216b 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -21,7 +21,7 @@ jobs:
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
-      - uses: actions/cache@v1
+      - uses: actions/cache@v3
         env:
           cache-name: cache-artifacts
         with:

From da193e5a148dd4ab3db99559db3d8414a45f837f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Mar 2023 13:03:17 +0000
Subject: [PATCH 096/375] Bump codecov/codecov-action from 1 to 3

Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 1 to 3.
- [Release notes](https://github.com/codecov/codecov-action/releases)
- [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/codecov/codecov-action/compare/v1...v3)

---
updated-dependencies:
- dependency-name: codecov/codecov-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 lib/SimpleNonlinearSolve/.github/workflows/CI.yml         | 2 +-
 lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index 80a2aea7b..6d7dedf46 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -37,6 +37,6 @@ jobs:
           GROUP: ${{ matrix.group }}
           JULIA_NUM_THREADS: 11
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@v3
         with:
           file: lcov.info
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
index 6abfcfddc..8215d41a5 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
@@ -54,6 +54,6 @@ jobs:
             exit(0)  # Exit immediately, as a success
           end
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@v3
         with:
           file: lcov.info

From 81cfc3771cfb36ba9db3c0909347d42bcf206462 Mon Sep 17 00:00:00 2001
From: Huiyu Xie <huiyuxie@Huiyus-MacBook-Pro.local>
Date: Fri, 24 Mar 2023 16:55:26 -0700
Subject: [PATCH 097/375] bracket

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 29 +++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 lib/SimpleNonlinearSolve/src/alefeld.jl

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
new file mode 100644
index 000000000..020b17f8a
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -0,0 +1,29 @@
+#struct Alefeld <: AbstractSimpleNonlinearSolveAlgorithm end
+
+# Define subrotine function bracket, check d to see whether the zero is found when using.
+function _bracket(f::Function, a, b, c)
+    fc = f(c)
+    if fc == 0
+        ā, b̄, d = a, b, c
+    else
+        fa, fb = f(a), f(b)
+        if fa * fc < 0 
+            ā, b̄, d = a, c, b
+        elseif fb * fc < 0
+            ā, b̄, d = c, b, a
+        end
+    end
+    return ā, b̄, d 
+end
+
+# Define subrotine function 
+#function _newton_quadratic()
+
+
+
+# test 
+function fk(x)
+    return 2 * x
+end
+
+_bracket(fk, -2, 2, 0)
\ No newline at end of file

From f2191d60247bd4c6aede40e5a8e8d7c10dbab63c Mon Sep 17 00:00:00 2001
From: Huiyu Xie <huiyuxie@Huiyus-MacBook-Pro.local>
Date: Fri, 24 Mar 2023 18:20:23 -0700
Subject: [PATCH 098/375] newton

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 35 +++++++++++++++++--------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 020b17f8a..9f5e2670d 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -1,29 +1,42 @@
 #struct Alefeld <: AbstractSimpleNonlinearSolveAlgorithm end
 
-# Define subrotine function bracket, check d to see whether the zero is found when using.
+# Define subrotine function bracket, check d to see whether the zero is found.
 function _bracket(f::Function, a, b, c)
-    fc = f(c)
-    if fc == 0
+    if f(c) == 0
         ā, b̄, d = a, b, c
     else
-        fa, fb = f(a), f(b)
-        if fa * fc < 0 
+        if f(a) * f(c) < 0 
             ā, b̄, d = a, c, b
-        elseif fb * fc < 0
+        elseif f(b) * f(c) < 0
             ā, b̄, d = c, b, a
         end
     end
     return ā, b̄, d 
 end
 
-# Define subrotine function 
-#function _newton_quadratic()
-
+# Define subrotine function newton quadratic, return the approximation of unique zero.
+function _newton_quadratic(f::Function, a, b, d, k)
+    A = ((f(b) - f(d)) / (b - d) - (f(a) - f(b)) / (a - b)) / (d - a) 
+    B = (f(b) - f(a)) / (b - a)
+    if A == 0
+        return a - (1 / B) * f(a)
+    elseif A * f(a) > 0
+        rᵢ₋₁ = a 
+    else 
+        rᵢ₋₁ = b
+    end 
+    for i in 1:k
+        rᵢ = rᵢ₋₁ - B * rᵢ₋₁ / (B + A * (2 * rᵢ₋₁ - a - b))
+        rᵢ₋₁ = rᵢ
+    end
+    return rᵢ₋₁
+end
 
 
 # test 
 function fk(x)
-    return 2 * x
+    return x^3
 end
 
-_bracket(fk, -2, 2, 0)
\ No newline at end of file
+_newton_quadratic(fk, -2, 4, 100, 2)
+

From 9fad692d07d92ba10d4d35585da1bec12747c6b1 Mon Sep 17 00:00:00 2001
From: Huiyu Xie <huiyuxie@Huiyus-MacBook-Pro.local>
Date: Fri, 24 Mar 2023 20:34:47 -0700
Subject: [PATCH 099/375] ipzero

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 9f5e2670d..5e39281c5 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -11,13 +11,15 @@ function _bracket(f::Function, a, b, c)
             ā, b̄, d = c, b, a
         end
     end
+
     return ā, b̄, d 
 end
 
-# Define subrotine function newton quadratic, return the approximation of unique zero.
+# Define subrotine function newton quadratic, return the approximation of zero.
 function _newton_quadratic(f::Function, a, b, d, k)
     A = ((f(b) - f(d)) / (b - d) - (f(a) - f(b)) / (a - b)) / (d - a) 
     B = (f(b) - f(a)) / (b - a)
+
     if A == 0
         return a - (1 / B) * f(a)
     elseif A * f(a) > 0
@@ -25,13 +27,29 @@ function _newton_quadratic(f::Function, a, b, d, k)
     else 
         rᵢ₋₁ = b
     end 
+
     for i in 1:k
         rᵢ = rᵢ₋₁ - B * rᵢ₋₁ / (B + A * (2 * rᵢ₋₁ - a - b))
         rᵢ₋₁ = rᵢ
     end
+
     return rᵢ₋₁
 end
 
+# Define subrotine function ipzero, also return the approximation of zero.
+function _ipzero(f::Function, a, b, c, d)
+    Q₁₁ = (c - d) * f(c) / (f(d) - f(c))
+    Q₂₁ = (b - c) * f(b) / (f(c) - f(b))
+    Q₃₁ = (a - b) * f(a) / (f(b) - f(a))
+    D₂₁ = (b - c) * f(c) / (f(c) - f(b))
+    D₃₁ = (a - b) * f(b) / (f(b) - f(a))
+    Q₂₂ = (D₂₁ - Q₁₁) * f(b) / (f(d) - f(b))
+    Q₃₂ = (D₃₁ - Q₂₁) * f(a) / (f(c) - f(a))
+    D₃₂ = (D₃₁ - Q₂₁) * f(c) / (f(c) - f(a))
+    Q₃₃ = (D₃₂ - Q₂₂) * f(a) / (f(d) - f(a))
+
+    return a + Q₃₁ + Q₃₂ + Q₃₃
+end
 
 # test 
 function fk(x)

From d2c82b214de0230225b26e125efd66d02a53527d Mon Sep 17 00:00:00 2001
From: Huiyu Xie <huiyuxie@Huiyus-MacBook-Pro.local>
Date: Sat, 25 Mar 2023 23:55:18 -0700
Subject: [PATCH 100/375] add solve

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 125 +++++++++++++++++++++---
 1 file changed, 110 insertions(+), 15 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 5e39281c5..ce726ea15 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -1,8 +1,111 @@
-#struct Alefeld <: AbstractSimpleNonlinearSolveAlgorithm end
+struct Alefeld <: AbstractBracketingAlgorithm end
 
-# Define subrotine function bracket, check d to see whether the zero is found.
+function SciMLBase.__solve(prob::NonlinearProblem,
+                            alg::Alefeld, args...; abstol = nothing,
+                            reltol = nothing,
+                            maxiters = 1000, kwargs...)
+                            
+    f = Base.Fix2(prob.f, prob.p)
+    a, b = prob.tspan
+    c = a - (b - a) / (f(b) - f(a)) * f(a)
+    
+    fc = f(c)
+    if iszero(fc)
+        return SciMLBase.build_solution(prob, alg, c, fc;
+                                        retcode = ReturnCode.Success, 
+                                        left = a,
+                                        right = b)
+    end
+    a, b, d = _bracket(f, a, b, c)
+    e = 0   # Set e as 0 before interation to avoid a non-value f(e)
+
+    for i in 2:maxiters
+        # The first bracketing block
+        f₁, f₂, f₃, f₄ = f(a), f(b), f(d), f(e)
+        if i == 2 || (f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄)
+            c = _newton_quadratic(f, a, b, d, 2)
+        else 
+            c = _ipzero(f, a, b, d, e)
+            if (c - a) * (c - b) ≥ 0
+                c = _newton_quadratic(f, a, b, d, 2)
+            end
+        end 
+        ē, fc = d, f(c)   
+        iszero(fc) &&
+            return SciMLBase.build_solution(prob, alg, c, fc;
+                                        retcode = ReturnCode.Success, 
+                                        left = a,
+                                        right = b)
+        ā, b̄, d̄ = _bracket(f, a, b, c) 
+
+        # The second bracketing block
+        f₁, f₂, f₃, f₄ = f(ā), f(b̄), f(d̄), f(ē)
+        if f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄
+            c = _newton_quadratic(f, ā, b̄, d̄, 3)
+        else 
+            c = _ipzero(f, ā, b̄, d̄, ē)
+            if (c - ā) * (c - b̄) ≥ 0
+                c = _newton_quadratic(f, ā, b̄, d̄, 3)
+            end
+        end
+        fc = f(c)
+        iszero(fc) &&
+            return SciMLBase.build_solution(prob, alg, c, fc;
+                                        retcode = ReturnCode.Success, 
+                                        left = a,
+                                        right = b)
+        ā, b̄, d̄ = _bracket(f, ā, b̄, c) 
+
+        # The third bracketing block
+        if abs(f(ā)) < abs(f(b̄))
+            u = ā
+        else
+            u = b̄
+        end
+        c = u - 2 * (b̄ - ā) / (f(b̄) - f(ā)) * f(u)
+        if (abs(c - u)) > 0.5 * (b̄ - ā)
+            c = 0.5 * (ā + b̄)
+        end
+        fc = f(c)
+        iszero(fc) &&
+            return SciMLBase.build_solution(prob, alg, c, fc;
+                                        retcode = ReturnCode.Success, 
+                                        left = a,
+                                        right = b)
+        ā, b̄, d = _bracket(f, ā, b̄, c) 
+
+        # The last bracketing block
+        if b̄ - ā < 0.5 * (b - a)
+            a, b, e = ā, b̄, d̄
+        else
+            e = d
+            c = 0.5 * (ā + b̄)
+            fc = f(c)
+            iszero(fc) &&
+            return SciMLBase.build_solution(prob, alg, c, fc;
+                                        retcode = ReturnCode.Success, 
+                                        left = a,
+                                        right = b)
+            a, b, d = _bracket(f, ā, b̄, c)
+        end
+    end
+
+    # Reassign the value a, b, and c
+    if b == c
+        b = d
+    elseif a == c
+        a = d
+    end
+    fc = f(c)
+
+    # Reuturn solution when run out of max interation
+    return SciMLBase.build_solution(prob, alg, c, fc; retcode = ReturnCode.MaxIters,
+                                    left = a, right = b)
+end
+
+# Define subrotine function bracket, check fc before bracket to return solution
 function _bracket(f::Function, a, b, c)
-    if f(c) == 0
+    if iszero(f(c))
         ā, b̄, d = a, b, c
     else
         if f(a) * f(c) < 0 
@@ -15,12 +118,12 @@ function _bracket(f::Function, a, b, c)
     return ā, b̄, d 
 end
 
-# Define subrotine function newton quadratic, return the approximation of zero.
+# Define subrotine function newton quadratic, return the approximation of zero
 function _newton_quadratic(f::Function, a, b, d, k)
     A = ((f(b) - f(d)) / (b - d) - (f(a) - f(b)) / (a - b)) / (d - a) 
     B = (f(b) - f(a)) / (b - a)
 
-    if A == 0
+    if iszero(A)
         return a - (1 / B) * f(a)
     elseif A * f(a) > 0
         rᵢ₋₁ = a 
@@ -36,7 +139,7 @@ function _newton_quadratic(f::Function, a, b, d, k)
     return rᵢ₋₁
 end
 
-# Define subrotine function ipzero, also return the approximation of zero.
+# Define subrotine function ipzero, also return the approximation of zero
 function _ipzero(f::Function, a, b, c, d)
     Q₁₁ = (c - d) * f(c) / (f(d) - f(c))
     Q₂₁ = (b - c) * f(b) / (f(c) - f(b))
@@ -49,12 +152,4 @@ function _ipzero(f::Function, a, b, c, d)
     Q₃₃ = (D₃₂ - Q₂₂) * f(a) / (f(d) - f(a))
 
     return a + Q₃₁ + Q₃₂ + Q₃₃
-end
-
-# test 
-function fk(x)
-    return x^3
-end
-
-_newton_quadratic(fk, -2, 4, 100, 2)
-
+end
\ No newline at end of file

From 918dd1dd345a5546ee94e317a275d8644c6b5e59 Mon Sep 17 00:00:00 2001
From: Huiyu <xhy19127@gmail.com>
Date: Sun, 26 Mar 2023 00:00:04 -0700
Subject: [PATCH 101/375] solve

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index ce726ea15..1d9b9bfef 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -17,8 +17,9 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                                         right = b)
     end
     a, b, d = _bracket(f, a, b, c)
-    e = 0   # Set e as 0 before interation to avoid a non-value f(e)
+    e = 0   # Set e as 0 before iteration to avoid a non-value f(e)
 
+    # Begin algorithm iteration
     for i in 2:maxiters
         # The first bracketing block
         f₁, f₂, f₃, f₄ = f(a), f(b), f(d), f(e)

From 43b0902ba41dbba6799a7ea282598e264adaf10d Mon Sep 17 00:00:00 2001
From: Huiyu <huiyuxie.sde@gmail.com>
Date: Sun, 26 Mar 2023 00:07:10 -0700
Subject: [PATCH 102/375] comment

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 1d9b9bfef..e58005b4a 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -1,3 +1,7 @@
+'''
+# add annotation here
+'''
+
 struct Alefeld <: AbstractBracketingAlgorithm end
 
 function SciMLBase.__solve(prob::NonlinearProblem,

From b1195953732557f546b6e3004456bc1c4db49aa1 Mon Sep 17 00:00:00 2001
From: Huiyu <huiyuxie.sde@gmail.com>
Date: Sun, 26 Mar 2023 21:16:44 -0700
Subject: [PATCH 103/375] add alefeld

---
 lib/SimpleNonlinearSolve/Project.toml            | 16 ++++++++++------
 .../src/SimpleNonlinearSolve.jl                  |  5 +++--
 lib/SimpleNonlinearSolve/src/alefeld.jl          | 15 ++++++++++-----
 lib/SimpleNonlinearSolve/test/basictests.jl      | 12 ++++++++++++
 4 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 55ccee5c4..b34c4387c 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -5,22 +5,20 @@ version = "0.1.14"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
+SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
-[weakdeps]
-NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
-
-[extensions]
-SimpleBatchedNonlinearSolveExt = "NNlib"
-
 [compat]
 ArrayInterface = "6, 7"
 DiffEqBase = "6.119"
@@ -34,6 +32,9 @@ SnoopPrecompile = "1"
 StaticArraysCore = "1.4"
 julia = "1.6"
 
+[extensions]
+SimpleBatchedNonlinearSolveExt = "NNlib"
+
 [extras]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
@@ -44,3 +45,6 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
 test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib"]
+
+[weakdeps]
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 4c90b094f..a33dd9297 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -38,6 +38,7 @@ include("brent.jl")
 include("dfsane.jl")
 include("ad.jl")
 include("halley.jl")
+include("alefeld.jl")
 
 import SnoopPrecompile
 
@@ -59,13 +60,13 @@ SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
     =#
 
     prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p, T.((0.0, 2.0)), T(2))
-    for alg in (Bisection, Falsi, Ridder, Brent)
+    for alg in (Bisection, Falsi, Ridder, Brent, Alefeld)
         solve(prob_brack, alg(), abstol = T(1e-2))
     end
 end end
 
 # DiffEq styled algorithms
 export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
-       Ridder, SimpleNewtonRaphson, SimpleTrustRegion
+       Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index e58005b4a..1164875e7 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -1,10 +1,15 @@
-'''
-# add annotation here
-'''
+"""
+`Alefeld()` 
+
+An implementation of algorithm 4.2 from [Alefeld](https://dl.acm.org/doi/10.1145/210089.210111).
+
+The paper brought up two new algorithms. Here choose to implement algorithm 4.2 rather than 
+algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal procedure.
+"""
 
 struct Alefeld <: AbstractBracketingAlgorithm end
 
-function SciMLBase.__solve(prob::NonlinearProblem,
+function SciMLBase.solve(prob::NonlinearProblem,
                             alg::Alefeld, args...; abstol = nothing,
                             reltol = nothing,
                             maxiters = 1000, kwargs...)
@@ -23,7 +28,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
     a, b, d = _bracket(f, a, b, c)
     e = 0   # Set e as 0 before iteration to avoid a non-value f(e)
 
-    # Begin algorithm iteration
+    # Begin of algorithm iteration
     for i in 2:maxiters
         # The first bracketing block
         f₁, f₂, f₃, f₄ = f(a), f(b), f(d), f(e)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 6906b6f2d..5b76ce15f 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -210,6 +210,18 @@ for p in 1.1:0.1:100.0
     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
 end
 
+# Alefeld
+g = function (p)
+    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
+    sol = solve(probN, Alefeld())
+    return sol.u
+end
+
+for p in 1.1:0.1:100.0
+    @test g(p) ≈ sqrt(p)
+    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+end
+
 f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
 t = (p) -> [sqrt(p[2] / p[1])]
 p = [0.9, 50.0]

From a81aa30389d6235a4ed6b3bb2c0a6fe40883dc7b Mon Sep 17 00:00:00 2001
From: Huiyu Xie <huiyuxie.sde@gmail.com>
Date: Wed, 29 Mar 2023 09:14:51 -0700
Subject: [PATCH 104/375] Update src/alefeld.jl

Co-authored-by: Christopher Rackauckas <accounts@chrisrackauckas.com>
---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 1164875e7..93d72d514 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -150,7 +150,7 @@ function _newton_quadratic(f::Function, a, b, d, k)
 end
 
 # Define subrotine function ipzero, also return the approximation of zero
-function _ipzero(f::Function, a, b, c, d)
+function _ipzero(f::F, a, b, c, d) where F
     Q₁₁ = (c - d) * f(c) / (f(d) - f(c))
     Q₂₁ = (b - c) * f(b) / (f(c) - f(b))
     Q₃₁ = (a - b) * f(a) / (f(b) - f(a))

From f618dfccb81811f25f9e8b3ab8b552ee97789b62 Mon Sep 17 00:00:00 2001
From: Huiyu Xie <huiyuxie.sde@gmail.com>
Date: Wed, 29 Mar 2023 09:20:32 -0700
Subject: [PATCH 105/375] Update src/alefeld.jl

Co-authored-by: Christopher Rackauckas <accounts@chrisrackauckas.com>
---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 93d72d514..1f53df0c9 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -114,7 +114,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
 end
 
 # Define subrotine function bracket, check fc before bracket to return solution
-function _bracket(f::Function, a, b, c)
+function _bracket(f::F, a, b, c) where F
     if iszero(f(c))
         ā, b̄, d = a, b, c
     else
@@ -129,7 +129,7 @@ function _bracket(f::Function, a, b, c)
 end
 
 # Define subrotine function newton quadratic, return the approximation of zero
-function _newton_quadratic(f::Function, a, b, d, k)
+function _newton_quadratic(f::F, a, b, d, k) where F
     A = ((f(b) - f(d)) / (b - d) - (f(a) - f(b)) / (a - b)) / (d - a) 
     B = (f(b) - f(a)) / (b - a)
 

From 0bb2ee41586dd35bf48f642511ff6b1bd334befa Mon Sep 17 00:00:00 2001
From: Huiyu Xie <huiyuxie.sde@gmail.com>
Date: Wed, 29 Mar 2023 09:21:22 -0700
Subject: [PATCH 106/375] Update src/alefeld.jl

Co-authored-by: Christopher Rackauckas <accounts@chrisrackauckas.com>
---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 1f53df0c9..bcc78f50d 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -6,7 +6,6 @@ An implementation of algorithm 4.2 from [Alefeld](https://dl.acm.org/doi/10.1145
 The paper brought up two new algorithms. Here choose to implement algorithm 4.2 rather than 
 algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal procedure.
 """
-
 struct Alefeld <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::NonlinearProblem,

From 4cdcb84052c0b95cc8294021ae8328aa649d3668 Mon Sep 17 00:00:00 2001
From: huiyuxie <huiyuxie.sde@gmail.com>
Date: Wed, 29 Mar 2023 12:34:56 -0400
Subject: [PATCH 107/375] resolve

---
 lib/SimpleNonlinearSolve/Project.toml   | 18 +++++++-----------
 lib/SimpleNonlinearSolve/src/alefeld.jl |  7 +++----
 2 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index b34c4387c..0be404135 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -5,20 +5,22 @@ version = "0.1.14"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
-BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
-SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
+[weakdeps]
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+
+[extensions]
+SimpleBatchedNonlinearSolveExt = "NNlib"
+
 [compat]
 ArrayInterface = "6, 7"
 DiffEqBase = "6.119"
@@ -32,9 +34,6 @@ SnoopPrecompile = "1"
 StaticArraysCore = "1.4"
 julia = "1.6"
 
-[extensions]
-SimpleBatchedNonlinearSolveExt = "NNlib"
-
 [extras]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
@@ -44,7 +43,4 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib"]
-
-[weakdeps]
-NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib"]
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 1164875e7..bcc78f50d 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -6,7 +6,6 @@ An implementation of algorithm 4.2 from [Alefeld](https://dl.acm.org/doi/10.1145
 The paper brought up two new algorithms. Here choose to implement algorithm 4.2 rather than 
 algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal procedure.
 """
-
 struct Alefeld <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::NonlinearProblem,
@@ -114,7 +113,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
 end
 
 # Define subrotine function bracket, check fc before bracket to return solution
-function _bracket(f::Function, a, b, c)
+function _bracket(f::F, a, b, c) where F
     if iszero(f(c))
         ā, b̄, d = a, b, c
     else
@@ -129,7 +128,7 @@ function _bracket(f::Function, a, b, c)
 end
 
 # Define subrotine function newton quadratic, return the approximation of zero
-function _newton_quadratic(f::Function, a, b, d, k)
+function _newton_quadratic(f::F, a, b, d, k) where F
     A = ((f(b) - f(d)) / (b - d) - (f(a) - f(b)) / (a - b)) / (d - a) 
     B = (f(b) - f(a)) / (b - a)
 
@@ -150,7 +149,7 @@ function _newton_quadratic(f::Function, a, b, d, k)
 end
 
 # Define subrotine function ipzero, also return the approximation of zero
-function _ipzero(f::Function, a, b, c, d)
+function _ipzero(f::F, a, b, c, d) where F
     Q₁₁ = (c - d) * f(c) / (f(d) - f(c))
     Q₂₁ = (b - c) * f(b) / (f(c) - f(b))
     Q₃₁ = (a - b) * f(a) / (f(b) - f(a))

From 093e901d06ece01db3d1cf21e0b8f5aa2a11fb88 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 29 Mar 2023 16:12:42 -0400
Subject: [PATCH 108/375] Update src/alefeld.jl

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index bcc78f50d..9bad1eadc 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -8,7 +8,7 @@ algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal
 """
 struct Alefeld <: AbstractBracketingAlgorithm end
 
-function SciMLBase.solve(prob::NonlinearProblem,
+function SciMLBase.solve(prob::IntervalNonlinearProblem,
                             alg::Alefeld, args...; abstol = nothing,
                             reltol = nothing,
                             maxiters = 1000, kwargs...)

From 76d04fc63ad5278cc4161f963b21efe5f1796710 Mon Sep 17 00:00:00 2001
From: huiyuxie <huiyuxie.sde@gmail.com>
Date: Wed, 29 Mar 2023 17:11:31 -0400
Subject: [PATCH 109/375] fix

---
 lib/SimpleNonlinearSolve/src/alefeld.jl |   5 +-
 lib/SimpleNonlinearSolve/src/test.jl    | 168 ++++++++++++++++++++++++
 2 files changed, 171 insertions(+), 2 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/test.jl

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index bcc78f50d..2c11a2749 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -8,7 +8,7 @@ algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal
 """
 struct Alefeld <: AbstractBracketingAlgorithm end
 
-function SciMLBase.solve(prob::NonlinearProblem,
+function SciMLBase.solve(prob::IntervalNonlinearProblem,
                             alg::Alefeld, args...; abstol = nothing,
                             reltol = nothing,
                             maxiters = 1000, kwargs...)
@@ -16,6 +16,7 @@ function SciMLBase.solve(prob::NonlinearProblem,
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     c = a - (b - a) / (f(b) - f(a)) * f(a)
+    @show c
     
     fc = f(c)
     if iszero(fc)
@@ -129,7 +130,7 @@ end
 
 # Define subrotine function newton quadratic, return the approximation of zero
 function _newton_quadratic(f::F, a, b, d, k) where F
-    A = ((f(b) - f(d)) / (b - d) - (f(a) - f(b)) / (a - b)) / (d - a) 
+    A = ((f(d) - f(b)) / (d - b) - (f(b) - f(a)) / (b - a)) / (d - a) 
     B = (f(b) - f(a)) / (b - a)
 
     if iszero(A)
diff --git a/lib/SimpleNonlinearSolve/src/test.jl b/lib/SimpleNonlinearSolve/src/test.jl
new file mode 100644
index 000000000..ce543dede
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/test.jl
@@ -0,0 +1,168 @@
+using SimpleNonlinearSolve
+using StaticArrays
+using BenchmarkTools
+using DiffEqBase
+using Test
+
+function test(f::Function, a, b) 
+    c = a - (b - a) / (f(b) - f(a)) * f(a)
+    println("0  ", c)
+    
+    fc = f(c)
+    if iszero(fc)
+        return c
+    end
+    a, b, d = _bracket(f, a, b, c)
+    println("a   ", a, "b   ", b, "d   ", d)
+    e = 0   # Set e as 0 before iteration to avoid a non-value f(e)
+
+    # Begin of algorithm iteration
+    for i in 2:1000
+        # The first bracketing block
+        f₁, f₂, f₃, f₄ = f(a), f(b), f(d), f(e)
+        if i == 2 || (f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄)
+            c = _newton_quadratic(f, a, b, d, 2)
+            println("1 ", "a   ", a, "b   ", b, "c   ", c)
+        else 
+            c = _ipzero(f, a, b, d, e)
+            if (c - a) * (c - b) ≥ 0
+                c = _newton_quadratic(f, a, b, d, 2)
+            end
+        end 
+        ē, fc = d, f(c)   
+        iszero(fc) &&
+            return c
+        ā, b̄, d̄ = _bracket(f, a, b, c) 
+
+        # The second bracketing block
+        f₁, f₂, f₃, f₄ = f(ā), f(b̄), f(d̄), f(ē)
+        if f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄
+            c = _newton_quadratic(f, ā, b̄, d̄, 3)
+        else 
+            c = _ipzero(f, ā, b̄, d̄, ē)
+            if (c - ā) * (c - b̄) ≥ 0
+                c = _newton_quadratic(f, ā, b̄, d̄, 3)
+            end
+        end
+        fc = f(c)
+        iszero(fc) &&
+            return c
+        ā, b̄, d̄ = _bracket(f, ā, b̄, c) 
+
+        # The third bracketing block
+        if abs(f(ā)) < abs(f(b̄))
+            u = ā
+        else
+            u = b̄
+        end
+        c = u - 2 * (b̄ - ā) / (f(b̄) - f(ā)) * f(u)
+        if (abs(c - u)) > 0.5 * (b̄ - ā)
+            c = 0.5 * (ā + b̄)
+        end
+        fc = f(c)
+        iszero(fc) &&
+            return c
+        ā, b̄, d = _bracket(f, ā, b̄, c) 
+
+        # The last bracketing block
+        if b̄ - ā < 0.5 * (b - a)
+            a, b, e = ā, b̄, d̄
+        else
+            e = d
+            c = 0.5 * (ā + b̄)
+            fc = f(c)
+            iszero(fc) &&
+            return c
+            a, b, d = _bracket(f, ā, b̄, c)
+        end
+        println("i   ", i)
+    end
+
+    # Reassign the value a, b, and c
+    if b == c
+        b = d
+    elseif a == c
+        a = d
+    end
+    fc = f(c)
+
+    # Reuturn solution when run out of max interation
+    return c
+end
+
+# Define subrotine function bracket, check fc before bracket to return solution
+function _bracket(f::F, a, b, c) where F
+    if iszero(f(c))
+        ā, b̄, d = a, b, c
+    else
+        if f(a) * f(c) < 0 
+            ā, b̄, d = a, c, b
+        elseif f(b) * f(c) < 0
+            ā, b̄, d = c, b, a
+        end
+    end
+
+    return ā, b̄, d 
+end
+
+# Define subrotine function newton quadratic, return the approximation of zero
+function _newton_quadratic(f::F, a, b, d, k) where F
+    A = ((f(d) - f(b)) / (d - b) - (f(b) - f(a)) / (b - a)) / (d - a) 
+    B = (f(b) - f(a)) / (b - a)
+    println("A   ", A)
+    println("B   ", B)
+
+
+    if iszero(A)
+        return a - (1 / B) * f(a)
+    elseif A * f(a) > 0
+        rᵢ₋₁ = a 
+    else 
+        rᵢ₋₁ = b    
+    end 
+
+    for i in 1:k
+        rᵢ = rᵢ₋₁ - B * rᵢ₋₁ / (B + A * (2 * rᵢ₋₁ - a - b))
+        rᵢ₋₁ = rᵢ
+    end
+
+    return rᵢ₋₁
+end
+
+# Define subrotine function ipzero, also return the approximation of zero
+function _ipzero(f::F, a, b, c, d) where F
+    Q₁₁ = (c - d) * f(c) / (f(d) - f(c))
+    Q₂₁ = (b - c) * f(b) / (f(c) - f(b))
+    Q₃₁ = (a - b) * f(a) / (f(b) - f(a))
+    D₂₁ = (b - c) * f(c) / (f(c) - f(b))
+    D₃₁ = (a - b) * f(b) / (f(b) - f(a))
+    Q₂₂ = (D₂₁ - Q₁₁) * f(b) / (f(d) - f(b))
+    Q₃₂ = (D₃₁ - Q₂₁) * f(a) / (f(c) - f(a))
+    D₃₂ = (D₃₁ - Q₂₁) * f(c) / (f(c) - f(a))
+    Q₃₃ = (D₃₂ - Q₂₂) * f(a) / (f(d) - f(a))
+
+    return a + Q₃₁ + Q₃₂ + Q₃₃
+end
+
+
+
+
+
+
+function f0(x)
+    return x^2 - 1.1
+end
+
+a = 1.0
+b = 20.0
+
+#= global i = 0
+for p in 1:100
+    println(i)
+    test(f0, a, b)
+    #@test g(p) ≈ sqrt(p)
+    #@test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+    global i += 1
+end =#
+
+test(f0, a, b)
\ No newline at end of file

From 76d3904cabf122beb07a254baff753ba8f721e9e Mon Sep 17 00:00:00 2001
From: huiyuxie <huiyuxie.sde@gmail.com>
Date: Wed, 29 Mar 2023 21:30:21 -0400
Subject: [PATCH 110/375] clean test

---
 lib/SimpleNonlinearSolve/src/test.jl | 168 ---------------------------
 1 file changed, 168 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/src/test.jl

diff --git a/lib/SimpleNonlinearSolve/src/test.jl b/lib/SimpleNonlinearSolve/src/test.jl
deleted file mode 100644
index ce543dede..000000000
--- a/lib/SimpleNonlinearSolve/src/test.jl
+++ /dev/null
@@ -1,168 +0,0 @@
-using SimpleNonlinearSolve
-using StaticArrays
-using BenchmarkTools
-using DiffEqBase
-using Test
-
-function test(f::Function, a, b) 
-    c = a - (b - a) / (f(b) - f(a)) * f(a)
-    println("0  ", c)
-    
-    fc = f(c)
-    if iszero(fc)
-        return c
-    end
-    a, b, d = _bracket(f, a, b, c)
-    println("a   ", a, "b   ", b, "d   ", d)
-    e = 0   # Set e as 0 before iteration to avoid a non-value f(e)
-
-    # Begin of algorithm iteration
-    for i in 2:1000
-        # The first bracketing block
-        f₁, f₂, f₃, f₄ = f(a), f(b), f(d), f(e)
-        if i == 2 || (f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄)
-            c = _newton_quadratic(f, a, b, d, 2)
-            println("1 ", "a   ", a, "b   ", b, "c   ", c)
-        else 
-            c = _ipzero(f, a, b, d, e)
-            if (c - a) * (c - b) ≥ 0
-                c = _newton_quadratic(f, a, b, d, 2)
-            end
-        end 
-        ē, fc = d, f(c)   
-        iszero(fc) &&
-            return c
-        ā, b̄, d̄ = _bracket(f, a, b, c) 
-
-        # The second bracketing block
-        f₁, f₂, f₃, f₄ = f(ā), f(b̄), f(d̄), f(ē)
-        if f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄
-            c = _newton_quadratic(f, ā, b̄, d̄, 3)
-        else 
-            c = _ipzero(f, ā, b̄, d̄, ē)
-            if (c - ā) * (c - b̄) ≥ 0
-                c = _newton_quadratic(f, ā, b̄, d̄, 3)
-            end
-        end
-        fc = f(c)
-        iszero(fc) &&
-            return c
-        ā, b̄, d̄ = _bracket(f, ā, b̄, c) 
-
-        # The third bracketing block
-        if abs(f(ā)) < abs(f(b̄))
-            u = ā
-        else
-            u = b̄
-        end
-        c = u - 2 * (b̄ - ā) / (f(b̄) - f(ā)) * f(u)
-        if (abs(c - u)) > 0.5 * (b̄ - ā)
-            c = 0.5 * (ā + b̄)
-        end
-        fc = f(c)
-        iszero(fc) &&
-            return c
-        ā, b̄, d = _bracket(f, ā, b̄, c) 
-
-        # The last bracketing block
-        if b̄ - ā < 0.5 * (b - a)
-            a, b, e = ā, b̄, d̄
-        else
-            e = d
-            c = 0.5 * (ā + b̄)
-            fc = f(c)
-            iszero(fc) &&
-            return c
-            a, b, d = _bracket(f, ā, b̄, c)
-        end
-        println("i   ", i)
-    end
-
-    # Reassign the value a, b, and c
-    if b == c
-        b = d
-    elseif a == c
-        a = d
-    end
-    fc = f(c)
-
-    # Reuturn solution when run out of max interation
-    return c
-end
-
-# Define subrotine function bracket, check fc before bracket to return solution
-function _bracket(f::F, a, b, c) where F
-    if iszero(f(c))
-        ā, b̄, d = a, b, c
-    else
-        if f(a) * f(c) < 0 
-            ā, b̄, d = a, c, b
-        elseif f(b) * f(c) < 0
-            ā, b̄, d = c, b, a
-        end
-    end
-
-    return ā, b̄, d 
-end
-
-# Define subrotine function newton quadratic, return the approximation of zero
-function _newton_quadratic(f::F, a, b, d, k) where F
-    A = ((f(d) - f(b)) / (d - b) - (f(b) - f(a)) / (b - a)) / (d - a) 
-    B = (f(b) - f(a)) / (b - a)
-    println("A   ", A)
-    println("B   ", B)
-
-
-    if iszero(A)
-        return a - (1 / B) * f(a)
-    elseif A * f(a) > 0
-        rᵢ₋₁ = a 
-    else 
-        rᵢ₋₁ = b    
-    end 
-
-    for i in 1:k
-        rᵢ = rᵢ₋₁ - B * rᵢ₋₁ / (B + A * (2 * rᵢ₋₁ - a - b))
-        rᵢ₋₁ = rᵢ
-    end
-
-    return rᵢ₋₁
-end
-
-# Define subrotine function ipzero, also return the approximation of zero
-function _ipzero(f::F, a, b, c, d) where F
-    Q₁₁ = (c - d) * f(c) / (f(d) - f(c))
-    Q₂₁ = (b - c) * f(b) / (f(c) - f(b))
-    Q₃₁ = (a - b) * f(a) / (f(b) - f(a))
-    D₂₁ = (b - c) * f(c) / (f(c) - f(b))
-    D₃₁ = (a - b) * f(b) / (f(b) - f(a))
-    Q₂₂ = (D₂₁ - Q₁₁) * f(b) / (f(d) - f(b))
-    Q₃₂ = (D₃₁ - Q₂₁) * f(a) / (f(c) - f(a))
-    D₃₂ = (D₃₁ - Q₂₁) * f(c) / (f(c) - f(a))
-    Q₃₃ = (D₃₂ - Q₂₂) * f(a) / (f(d) - f(a))
-
-    return a + Q₃₁ + Q₃₂ + Q₃₃
-end
-
-
-
-
-
-
-function f0(x)
-    return x^2 - 1.1
-end
-
-a = 1.0
-b = 20.0
-
-#= global i = 0
-for p in 1:100
-    println(i)
-    test(f0, a, b)
-    #@test g(p) ≈ sqrt(p)
-    #@test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-    global i += 1
-end =#
-
-test(f0, a, b)
\ No newline at end of file

From 4d332029757c814a1cbfec26db610e7dc59eedac Mon Sep 17 00:00:00 2001
From: Huiyu Xie <huiyuxie.sde@gmail.com>
Date: Thu, 30 Mar 2023 09:00:45 -0700
Subject: [PATCH 111/375] Update src/alefeld.jl

Co-authored-by: Christopher Rackauckas <accounts@chrisrackauckas.com>
---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 2c11a2749..77fa30921 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -26,7 +26,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
                                         right = b)
     end
     a, b, d = _bracket(f, a, b, c)
-    e = 0   # Set e as 0 before iteration to avoid a non-value f(e)
+    e = zero(a)   # Set e as 0 before iteration to avoid a non-value f(e)
 
     # Begin of algorithm iteration
     for i in 2:maxiters

From 15b6c13745d70624076f35083c08595e216bab52 Mon Sep 17 00:00:00 2001
From: huiyuxie <huiyuxie.sde@gmail.com>
Date: Thu, 30 Mar 2023 13:47:56 -0400
Subject: [PATCH 112/375] fix bugs

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 43 ++++++++++++++++++-------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 2c11a2749..542bff8b6 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -16,7 +16,6 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     c = a - (b - a) / (f(b) - f(a)) * f(a)
-    @show c
     
     fc = f(c)
     if iszero(fc)
@@ -26,7 +25,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
                                         right = b)
     end
     a, b, d = _bracket(f, a, b, c)
-    e = 0   # Set e as 0 before iteration to avoid a non-value f(e)
+    e = zero(a)    # Set e as 0 before iteration to avoid a non-value f(e)
 
     # Begin of algorithm iteration
     for i in 2:maxiters
@@ -40,7 +39,12 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
                 c = _newton_quadratic(f, a, b, d, 2)
             end
         end 
-        ē, fc = d, f(c)   
+        ē, fc = d, f(c)
+        (a == c || b == c) && 
+            return SciMLBase.build_solution(prob, alg, c, fc;
+                                            retcode = ReturnCode.FloatingPointLimit,
+                                            left = a, 
+                                            right = b)   
         iszero(fc) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
                                         retcode = ReturnCode.Success, 
@@ -59,11 +63,16 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
             end
         end
         fc = f(c)
+        (ā == c || b̄ == c) && 
+            return SciMLBase.build_solution(prob, alg, c, fc;
+                                            retcode = ReturnCode.FloatingPointLimit,
+                                            left = ā, 
+                                            right = b̄)
         iszero(fc) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
                                         retcode = ReturnCode.Success, 
-                                        left = a,
-                                        right = b)
+                                        left = ā,
+                                        right = b̄)
         ā, b̄, d̄ = _bracket(f, ā, b̄, c) 
 
         # The third bracketing block
@@ -77,11 +86,16 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
             c = 0.5 * (ā + b̄)
         end
         fc = f(c)
+        (ā == c || b̄ == c) && 
+            return SciMLBase.build_solution(prob, alg, c, fc;
+                                            retcode = ReturnCode.FloatingPointLimit,
+                                            left = ā, 
+                                            right = b̄)
         iszero(fc) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
                                         retcode = ReturnCode.Success, 
-                                        left = a,
-                                        right = b)
+                                        left = ā, 
+                                        right = b̄)
         ā, b̄, d = _bracket(f, ā, b̄, c) 
 
         # The last bracketing block
@@ -91,11 +105,16 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
             e = d
             c = 0.5 * (ā + b̄)
             fc = f(c)
+            (ā == c || b̄ == c) && 
+                return SciMLBase.build_solution(prob, alg, c, fc;
+                                                retcode = ReturnCode.FloatingPointLimit,
+                                                left = ā, 
+                                                right = b̄)
             iszero(fc) &&
-            return SciMLBase.build_solution(prob, alg, c, fc;
-                                        retcode = ReturnCode.Success, 
-                                        left = a,
-                                        right = b)
+                return SciMLBase.build_solution(prob, alg, c, fc;
+                                                retcode = ReturnCode.Success, 
+                                                left = ā,
+                                                right = b̄)
             a, b, d = _bracket(f, ā, b̄, c)
         end
     end
@@ -142,7 +161,7 @@ function _newton_quadratic(f::F, a, b, d, k) where F
     end 
 
     for i in 1:k
-        rᵢ = rᵢ₋₁ - B * rᵢ₋₁ / (B + A * (2 * rᵢ₋₁ - a - b))
+        rᵢ = rᵢ₋₁ - (f(a) + B * (rᵢ₋₁ - a) + A * (rᵢ₋₁ - a) * (rᵢ₋₁ - b)) / (B + A * (2 * rᵢ₋₁ - a - b))
         rᵢ₋₁ = rᵢ
     end
 

From 755fec85bc0c5401884352359ad36c65c6146db1 Mon Sep 17 00:00:00 2001
From: huiyuxie <huiyuxie.sde@gmail.com>
Date: Thu, 30 Mar 2023 17:12:18 -0400
Subject: [PATCH 113/375] more tests

---
 lib/SimpleNonlinearSolve/src/alefeld.jl     |  8 +++++--
 lib/SimpleNonlinearSolve/test/basictests.jl | 25 +++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 47b01d9ce..f468add9f 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -18,12 +18,16 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
     c = a - (b - a) / (f(b) - f(a)) * f(a)
     
     fc = f(c)
-    if iszero(fc)
+    (a == c || b == c) && 
+        return SciMLBase.build_solution(prob, alg, c, fc;
+                                        retcode = ReturnCode.FloatingPointLimit,
+                                        left = a, 
+                                        right = b)
+    iszero(fc) &&
         return SciMLBase.build_solution(prob, alg, c, fc;
                                         retcode = ReturnCode.Success, 
                                         left = a,
                                         right = b)
-    end
     a, b, d = _bracket(f, a, b, c)
     e = zero(a)   # Set e as 0 before iteration to avoid a non-value f(e)
 
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 5b76ce15f..310ca0638 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -222,6 +222,18 @@ for p in 1.1:0.1:100.0
     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
 end
 
+f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
+t = (p) -> [sqrt(p[2] / p[1])]
+p = [0.9, 50.0]
+g = function (p)
+    probN = IntervalNonlinearProblem{false}(f, tspan, p)
+    sol = solve(probN, Alefeld())
+    return [sol.u]
+end
+
+@test g(p) ≈ [sqrt(p[2] / p[1])]
+@test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
+
 f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
 t = (p) -> [sqrt(p[2] / p[1])]
 p = [0.9, 50.0]
@@ -288,6 +300,7 @@ probB = IntervalNonlinearProblem(f, tspan)
 sol = solve(probB, Falsi())
 @test sol.left ≈ sqrt(2.0)
 
+# Bisection
 sol = solve(probB, Bisection())
 @test sol.left ≈ sqrt(2.0)
 
@@ -315,6 +328,18 @@ probB = IntervalNonlinearProblem(f, tspan)
 sol = solve(probB, Brent())
 @test sol.left ≈ sqrt(2.0)
 
+# Alefeld
+sol = solve(probB, Alefeld())
+@test sol.u ≈ sqrt(2.0)
+tspan = (sqrt(2.0), 10.0)
+probB = IntervalNonlinearProblem(f, tspan)
+sol = solve(probB, Alefeld())
+@test sol.u ≈ sqrt(2.0)
+tspan = (0.0, sqrt(2.0))
+probB = IntervalNonlinearProblem(f, tspan)
+sol = solve(probB, Alefeld())
+@test sol.u ≈ sqrt(2.0)
+
 # Garuntee Tests for Bisection
 f = function (u, p)
     if u < 2.0

From 9cca5371c4df904eb64c1558e00148bbfa969ee3 Mon Sep 17 00:00:00 2001
From: Hendrik Ranocha <mail@ranocha.de>
Date: Fri, 31 Mar 2023 10:43:55 +0200
Subject: [PATCH 114/375] format

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 85 +++++++++++++------------
 1 file changed, 43 insertions(+), 42 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 47b01d9ce..e573c59d1 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -9,18 +9,17 @@ algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal
 struct Alefeld <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem,
-                            alg::Alefeld, args...; abstol = nothing,
-                            reltol = nothing,
-                            maxiters = 1000, kwargs...)
-                            
+                         alg::Alefeld, args...; abstol = nothing,
+                         reltol = nothing,
+                         maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     c = a - (b - a) / (f(b) - f(a)) * f(a)
-    
+
     fc = f(c)
     if iszero(fc)
         return SciMLBase.build_solution(prob, alg, c, fc;
-                                        retcode = ReturnCode.Success, 
+                                        retcode = ReturnCode.Success,
                                         left = a,
                                         right = b)
     end
@@ -33,47 +32,47 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
         f₁, f₂, f₃, f₄ = f(a), f(b), f(d), f(e)
         if i == 2 || (f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄)
             c = _newton_quadratic(f, a, b, d, 2)
-        else 
+        else
             c = _ipzero(f, a, b, d, e)
             if (c - a) * (c - b) ≥ 0
                 c = _newton_quadratic(f, a, b, d, 2)
             end
-        end 
+        end
         ē, fc = d, f(c)
-        (a == c || b == c) && 
+        (a == c || b == c) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
                                             retcode = ReturnCode.FloatingPointLimit,
-                                            left = a, 
-                                            right = b)   
+                                            left = a,
+                                            right = b)
         iszero(fc) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
-                                        retcode = ReturnCode.Success, 
-                                        left = a,
-                                        right = b)
-        ā, b̄, d̄ = _bracket(f, a, b, c) 
+                                            retcode = ReturnCode.Success,
+                                            left = a,
+                                            right = b)
+        ā, b̄, d̄ = _bracket(f, a, b, c)
 
         # The second bracketing block
         f₁, f₂, f₃, f₄ = f(ā), f(b̄), f(d̄), f(ē)
         if f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄
             c = _newton_quadratic(f, ā, b̄, d̄, 3)
-        else 
+        else
             c = _ipzero(f, ā, b̄, d̄, ē)
             if (c - ā) * (c - b̄) ≥ 0
                 c = _newton_quadratic(f, ā, b̄, d̄, 3)
             end
         end
         fc = f(c)
-        (ā == c || b̄ == c) && 
+        (ā == c || b̄ == c) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
                                             retcode = ReturnCode.FloatingPointLimit,
-                                            left = ā, 
+                                            left = ā,
                                             right = b̄)
         iszero(fc) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
-                                        retcode = ReturnCode.Success, 
-                                        left = ā,
-                                        right = b̄)
-        ā, b̄, d̄ = _bracket(f, ā, b̄, c) 
+                                            retcode = ReturnCode.Success,
+                                            left = ā,
+                                            right = b̄)
+        ā, b̄, d̄ = _bracket(f, ā, b̄, c)
 
         # The third bracketing block
         if abs(f(ā)) < abs(f(b̄))
@@ -86,17 +85,17 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
             c = 0.5 * (ā + b̄)
         end
         fc = f(c)
-        (ā == c || b̄ == c) && 
+        (ā == c || b̄ == c) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
                                             retcode = ReturnCode.FloatingPointLimit,
-                                            left = ā, 
+                                            left = ā,
                                             right = b̄)
         iszero(fc) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
-                                        retcode = ReturnCode.Success, 
-                                        left = ā, 
-                                        right = b̄)
-        ā, b̄, d = _bracket(f, ā, b̄, c) 
+                                            retcode = ReturnCode.Success,
+                                            left = ā,
+                                            right = b̄)
+        ā, b̄, d = _bracket(f, ā, b̄, c)
 
         # The last bracketing block
         if b̄ - ā < 0.5 * (b - a)
@@ -105,14 +104,14 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
             e = d
             c = 0.5 * (ā + b̄)
             fc = f(c)
-            (ā == c || b̄ == c) && 
+            (ā == c || b̄ == c) &&
                 return SciMLBase.build_solution(prob, alg, c, fc;
                                                 retcode = ReturnCode.FloatingPointLimit,
-                                                left = ā, 
+                                                left = ā,
                                                 right = b̄)
             iszero(fc) &&
                 return SciMLBase.build_solution(prob, alg, c, fc;
-                                                retcode = ReturnCode.Success, 
+                                                retcode = ReturnCode.Success,
                                                 left = ā,
                                                 right = b̄)
             a, b, d = _bracket(f, ā, b̄, c)
@@ -133,35 +132,37 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
 end
 
 # Define subrotine function bracket, check fc before bracket to return solution
-function _bracket(f::F, a, b, c) where F
+function _bracket(f::F, a, b, c) where {F}
     if iszero(f(c))
         ā, b̄, d = a, b, c
     else
-        if f(a) * f(c) < 0 
+        if f(a) * f(c) < 0
             ā, b̄, d = a, c, b
         elseif f(b) * f(c) < 0
             ā, b̄, d = c, b, a
         end
     end
 
-    return ā, b̄, d 
+    return ā, b̄, d
 end
 
 # Define subrotine function newton quadratic, return the approximation of zero
-function _newton_quadratic(f::F, a, b, d, k) where F
-    A = ((f(d) - f(b)) / (d - b) - (f(b) - f(a)) / (b - a)) / (d - a) 
+function _newton_quadratic(f::F, a, b, d, k) where {F}
+    A = ((f(d) - f(b)) / (d - b) - (f(b) - f(a)) / (b - a)) / (d - a)
     B = (f(b) - f(a)) / (b - a)
 
     if iszero(A)
         return a - (1 / B) * f(a)
     elseif A * f(a) > 0
-        rᵢ₋₁ = a 
-    else 
+        rᵢ₋₁ = a
+    else
         rᵢ₋₁ = b
-    end 
+    end
 
     for i in 1:k
-        rᵢ = rᵢ₋₁ - (f(a) + B * (rᵢ₋₁ - a) + A * (rᵢ₋₁ - a) * (rᵢ₋₁ - b)) / (B + A * (2 * rᵢ₋₁ - a - b))
+        rᵢ = rᵢ₋₁ -
+             (f(a) + B * (rᵢ₋₁ - a) + A * (rᵢ₋₁ - a) * (rᵢ₋₁ - b)) /
+             (B + A * (2 * rᵢ₋₁ - a - b))
         rᵢ₋₁ = rᵢ
     end
 
@@ -169,7 +170,7 @@ function _newton_quadratic(f::F, a, b, d, k) where F
 end
 
 # Define subrotine function ipzero, also return the approximation of zero
-function _ipzero(f::F, a, b, c, d) where F
+function _ipzero(f::F, a, b, c, d) where {F}
     Q₁₁ = (c - d) * f(c) / (f(d) - f(c))
     Q₂₁ = (b - c) * f(b) / (f(c) - f(b))
     Q₃₁ = (a - b) * f(a) / (f(b) - f(a))
@@ -181,4 +182,4 @@ function _ipzero(f::F, a, b, c, d) where F
     Q₃₃ = (D₃₂ - Q₂₂) * f(a) / (f(d) - f(a))
 
     return a + Q₃₁ + Q₃₂ + Q₃₃
-end
\ No newline at end of file
+end

From 56c21013462f167c876ae645c6008e9be82e12cf Mon Sep 17 00:00:00 2001
From: Hendrik Ranocha <mail@ranocha.de>
Date: Fri, 31 Mar 2023 11:35:31 +0200
Subject: [PATCH 115/375] format

---
 lib/SimpleNonlinearSolve/src/alefeld.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index ad2604058..61861a4a7 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -17,10 +17,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
     c = a - (b - a) / (f(b) - f(a)) * f(a)
 
     fc = f(c)
-    (a == c || b == c) && 
+    (a == c || b == c) &&
         return SciMLBase.build_solution(prob, alg, c, fc;
                                         retcode = ReturnCode.FloatingPointLimit,
-                                        left = a, 
+                                        left = a,
                                         right = b)
     iszero(fc) &&
         return SciMLBase.build_solution(prob, alg, c, fc;

From 6c4cf4d69ffb2e5915a169b2dc74f97db0fb79cb Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 6 Apr 2023 11:32:47 -0400
Subject: [PATCH 116/375] Use a mutable struct instead of Dict for Safe
 termination

---
 lib/SimpleNonlinearSolve/.gitignore                            | 2 +-
 lib/SimpleNonlinearSolve/Project.toml                          | 2 +-
 lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl | 3 ++-
 lib/SimpleNonlinearSolve/src/broyden.jl                        | 3 ++-
 lib/SimpleNonlinearSolve/src/lbroyden.jl                       | 3 ++-
 5 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.gitignore b/lib/SimpleNonlinearSolve/.gitignore
index e4cfbfe81..42affce03 100644
--- a/lib/SimpleNonlinearSolve/.gitignore
+++ b/lib/SimpleNonlinearSolve/.gitignore
@@ -1,3 +1,3 @@
 Manifest.toml
-
+.vscode
 wip
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 0be404135..a0becaaca 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -23,7 +23,7 @@ SimpleBatchedNonlinearSolveExt = "NNlib"
 
 [compat]
 ArrayInterface = "6, 7"
-DiffEqBase = "6.119"
+DiffEqBase = "6.123.0"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 NNlib = "0.8"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
index 07c117f4f..e3f394c3b 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
@@ -59,7 +59,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
         error("Broyden currently doesn't support SAFE_BEST termination modes")
     end
 
-    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? Dict() : nothing
+    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
+              nothing
     termination_condition = tc(storage)
 
     xₙ = x
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index e8d339c4c..e46469619 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -48,7 +48,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{false}, args...;
         error("Broyden currently doesn't support SAFE_BEST termination modes")
     end
 
-    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? Dict() : nothing
+    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
+              nothing
     termination_condition = tc(storage)
 
     xₙ = x
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
index 6a3fc0947..6cca7c11e 100644
--- a/lib/SimpleNonlinearSolve/src/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -63,7 +63,8 @@ end
         error("LBroyden currently doesn't support SAFE_BEST termination modes")
     end
 
-    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? Dict() : nothing
+    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
+              nothing
     termination_condition = tc(storage)
 
     xₙ = x

From 0ba9b91f91bea5dd67e2b641532a8050cbe1e049 Mon Sep 17 00:00:00 2001
From: Tim Holy <tim.holy@gmail.com>
Date: Mon, 24 Apr 2023 16:38:40 -0500
Subject: [PATCH 117/375] Migrate from SnoopPrecompile to PrecompileTools

---
 lib/SimpleNonlinearSolve/Project.toml                | 4 ++--
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index a0becaaca..aeb58ac43 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -12,7 +12,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
-SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
+PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [weakdeps]
@@ -30,7 +30,7 @@ NNlib = "0.8"
 Reexport = "0.2, 1"
 Requires = "1"
 SciMLBase = "1.73"
-SnoopPrecompile = "1"
+PrecompileTools = "1"
 StaticArraysCore = "1.4"
 julia = "1.6"
 
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index a33dd9297..46f1ab3c4 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -40,9 +40,9 @@ include("ad.jl")
 include("halley.jl")
 include("alefeld.jl")
 
-import SnoopPrecompile
+import PrecompileTools
 
-SnoopPrecompile.@precompile_all_calls begin for T in (Float32, Float64)
+PrecompileTools.@compile_workload begin for T in (Float32, Float64)
     prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
     for alg in (SimpleNewtonRaphson, Halley, Broyden, Klement, SimpleTrustRegion,
                 SimpleDFSane)

From 0659f6a5eb9b352c24be53fbd7af98c20f2344ec Mon Sep 17 00:00:00 2001
From: Utkarsh <rajpututkarsh530@gmail.com>
Date: Fri, 19 May 2023 19:28:53 -0400
Subject: [PATCH 118/375] Add support for f.jac in SimpleNewtonRaphson

---
 lib/SimpleNonlinearSolve/src/raphson.jl     |  5 ++++-
 lib/SimpleNonlinearSolve/test/basictests.jl | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 57d66b005..8d2d9796c 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -60,7 +60,10 @@ function SciMLBase.__solve(prob::NonlinearProblem,
     end
 
     for i in 1:maxiters
-        if alg_autodiff(alg)
+        if DiffEqBase.has_jac(prob.f)
+            dfx = prob.f.jac(x, prob.p)
+            fx = f(x)
+        elseif alg_autodiff(alg)
             fx, dfx = value_derivative(f, x)
         elseif x isa AbstractArray
             fx = f(x)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 310ca0638..81a047e0a 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -2,6 +2,7 @@ using SimpleNonlinearSolve
 using StaticArrays
 using BenchmarkTools
 using DiffEqBase
+using LinearAlgebra
 using Test
 
 const BATCHED_BROYDEN_SOLVERS = Broyden[]
@@ -489,3 +490,17 @@ for alg in (BATCHED_BROYDEN_SOLVERS..., BATCHED_LBROYDEN_SOLVERS...)
     @test sol.retcode == ReturnCode.Success
     @test abs.(sol.u) ≈ sqrt.(p)
 end
+
+## Specified Jacobian
+
+f, u0 = (u, p) -> u .* u .- p, randn(3)
+
+f_jac(u, p) = begin diagm(2 * u) end
+
+p = [2.0, 1.0, 5.0];
+
+probN = NonlinearProblem(NonlinearFunction(f, jac = f_jac), u0, p)
+
+sol = solve(probN, SimpleNewtonRaphson())
+
+@test abs.(sol.u) ≈ sqrt.(p)

From 4411caef1c0740f8d7effdff1a9940233069e887 Mon Sep 17 00:00:00 2001
From: Utkarsh <rajpututkarsh530@gmail.com>
Date: Wed, 24 May 2023 15:54:24 -0400
Subject: [PATCH 119/375] Add custom jac in SimpleTrustRegion

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 6 +++++-
 lib/SimpleNonlinearSolve/test/basictests.jl | 9 +++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index edd143215..aa893ffc1 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -112,7 +112,11 @@ function SciMLBase.__solve(prob::NonlinearProblem,
            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
     rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
 
-    if alg_autodiff(alg)
+
+    if DiffEqBase.has_jac(prob.f)
+        ∇f = prob.f.jac(x, prob.p)
+        F = f(x)
+    elseif alg_autodiff(alg)
         F, ∇f = value_derivative(f, x)
     elseif x isa AbstractArray
         F = f(x)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 81a047e0a..1a0249b65 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -491,7 +491,7 @@ for alg in (BATCHED_BROYDEN_SOLVERS..., BATCHED_LBROYDEN_SOLVERS...)
     @test abs.(sol.u) ≈ sqrt.(p)
 end
 
-## Specified Jacobian
+## User specified Jacobian
 
 f, u0 = (u, p) -> u .* u .- p, randn(3)
 
@@ -501,6 +501,7 @@ p = [2.0, 1.0, 5.0];
 
 probN = NonlinearProblem(NonlinearFunction(f, jac = f_jac), u0, p)
 
-sol = solve(probN, SimpleNewtonRaphson())
-
-@test abs.(sol.u) ≈ sqrt.(p)
+for alg in (SimpleNewtonRaphson(), SimpleTrustRegion())
+    sol = solve(probN, alg)
+    @test abs.(sol.u) ≈ sqrt.(p)
+end

From b1d8bea57a35c4e7a035ea68508563d3334b43c3 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 26 May 2023 14:32:16 -0700
Subject: [PATCH 120/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index aeb58ac43..c1f21c72e 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.14"
+version = "0.1.15"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
@@ -43,4 +43,4 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib"]
\ No newline at end of file
+test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib"]

From 0ca6ab0b9efb4b71ae1719451a9858cc0b6ce83e Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 8 Jun 2023 15:08:39 -0400
Subject: [PATCH 121/375] Format and add a format CI

---
 .../.github/workflows/FormatPR.yml            | 29 ++++++++
 .../ext/SimpleBatchedNonlinearSolveExt.jl     |  4 +-
 .../src/SimpleNonlinearSolve.jl               | 46 +++++++------
 lib/SimpleNonlinearSolve/src/ad.jl            | 52 +++++++-------
 lib/SimpleNonlinearSolve/src/alefeld.jl       | 68 +++++++++----------
 lib/SimpleNonlinearSolve/src/bisection.jl     | 18 ++---
 lib/SimpleNonlinearSolve/src/brent.jl         | 18 ++---
 lib/SimpleNonlinearSolve/src/broyden.jl       |  8 +--
 lib/SimpleNonlinearSolve/src/dfsane.jl        | 12 ++--
 lib/SimpleNonlinearSolve/src/falsi.jl         | 18 ++---
 lib/SimpleNonlinearSolve/src/halley.jl        | 22 +++---
 lib/SimpleNonlinearSolve/src/klement.jl       | 14 ++--
 lib/SimpleNonlinearSolve/src/lbroyden.jl      | 20 +++---
 lib/SimpleNonlinearSolve/src/raphson.jl       | 10 +--
 lib/SimpleNonlinearSolve/src/ridder.jl        | 22 +++---
 lib/SimpleNonlinearSolve/src/trustRegion.jl   | 53 +++++++--------
 lib/SimpleNonlinearSolve/test/basictests.jl   | 60 ++++++++--------
 lib/SimpleNonlinearSolve/test/runtests.jl     | 10 +--
 18 files changed, 261 insertions(+), 223 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml
new file mode 100644
index 000000000..b0cb28af5
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml
@@ -0,0 +1,29 @@
+name: format-pr
+on:
+  schedule:
+    - cron: '0 0 * * *'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install JuliaFormatter and format
+        run: |
+          julia  -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter"))'
+          julia  -e 'using JuliaFormatter; format(".")'
+      # https://github.com/marketplace/actions/create-pull-request
+      # https://github.com/peter-evans/create-pull-request#reference-example
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: Format .jl files
+          title: 'Automatic JuliaFormatter.jl run'
+          branch: auto-juliaformatter-pr
+          delete-branch: true
+          labels: formatting, automated pr, no changelog
+      - name: Check outputs
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
index e3f394c3b..dd7628888 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
@@ -31,7 +31,7 @@ function _init_J_batched(x::AbstractMatrix{T}) where {T}
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
-                           abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
+    abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
     f = Base.Fix2(prob.f, prob.p)
@@ -74,7 +74,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
         J⁻¹Δfₙ = _batched_mul(J⁻¹, Δfₙ)
         J⁻¹ += _batched_mul(((Δxₙ .- J⁻¹Δfₙ) ./
                              (_batched_mul(_batch_transpose(Δxₙ), J⁻¹Δfₙ) .+ T(1e-5))),
-                            _batched_mul(_batch_transpose(Δxₙ), J⁻¹))
+            _batched_mul(_batch_transpose(Δxₙ), J⁻¹))
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
             return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 46f1ab3c4..8749aa72b 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -16,7 +16,9 @@ end
 
 function __init__()
     @static if !isdefined(Base, :get_extension)
-        @require NNlib="872c559c-99b0-510c-b3b7-b6c96a88d5cd" begin include("../ext/SimpleBatchedNonlinearSolveExt.jl") end
+        @require NNlib="872c559c-99b0-510c-b3b7-b6c96a88d5cd" begin
+            include("../ext/SimpleBatchedNonlinearSolveExt.jl")
+        end
     end
 end
 
@@ -42,31 +44,35 @@ include("alefeld.jl")
 
 import PrecompileTools
 
-PrecompileTools.@compile_workload begin for T in (Float32, Float64)
-    prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-    for alg in (SimpleNewtonRaphson, Halley, Broyden, Klement, SimpleTrustRegion,
-                SimpleDFSane)
-        solve(prob_no_brack, alg(), abstol = T(1e-2))
-    end
+PrecompileTools.@compile_workload begin
+    for T in (Float32, Float64)
+        prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
+        for alg in (SimpleNewtonRaphson, Halley, Broyden, Klement, SimpleTrustRegion,
+            SimpleDFSane)
+            solve(prob_no_brack, alg(), abstol = T(1e-2))
+        end
 
-    #=
-    for alg in (SimpleNewtonRaphson,)
-        for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
-            u0 = T.(.1)
-            probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
-            solve(probN, alg(), tol = T(1e-2))
+        #=
+        for alg in (SimpleNewtonRaphson,)
+            for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
+                u0 = T.(.1)
+                probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
+                solve(probN, alg(), tol = T(1e-2))
+            end
         end
-    end
-    =#
+        =#
 
-    prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p, T.((0.0, 2.0)), T(2))
-    for alg in (Bisection, Falsi, Ridder, Brent, Alefeld)
-        solve(prob_brack, alg(), abstol = T(1e-2))
+        prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
+            T.((0.0, 2.0)),
+            T(2))
+        for alg in (Bisection, Falsi, Ridder, Brent, Alefeld)
+            solve(prob_brack, alg(), abstol = T(1e-2))
+        end
     end
-end end
+end
 
 # DiffEq styled algorithms
 export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
-       Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld
+    Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index dbd90243a..009cd227b 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -29,50 +29,50 @@ function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
 end
 
 function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
-                                                iip,
-                                                <:Dual{T, V, P}},
-                         alg::AbstractSimpleNonlinearSolveAlgorithm,
-                         args...; kwargs...) where {iip, T, V, P}
+        iip,
+        <:Dual{T, V, P}},
+    alg::AbstractSimpleNonlinearSolveAlgorithm,
+    args...; kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
-                                    retcode = sol.retcode)
+        retcode = sol.retcode)
 end
 function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
-                                                iip,
-                                                <:AbstractArray{<:Dual{T, V, P}}},
-                         alg::AbstractSimpleNonlinearSolveAlgorithm, args...;
-                         kwargs...) where {iip, T, V, P}
+        iip,
+        <:AbstractArray{<:Dual{T, V, P}}},
+    alg::AbstractSimpleNonlinearSolveAlgorithm, args...;
+    kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
-                                    retcode = sol.retcode)
+        retcode = sol.retcode)
 end
 
 # avoid ambiguities
 for Alg in [Bisection]
     @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
-                                                                  <:Dual{T, V, P}},
-                                   alg::$Alg, args...;
-                                   kwargs...) where {uType, iip, T, V, P}
+            <:Dual{T, V, P}},
+        alg::$Alg, args...;
+        kwargs...) where {uType, iip, T, V, P}
         sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
         return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials),
-                                        sol.resid; retcode = sol.retcode,
-                                        left = Dual{T, V, P}(sol.left, partials),
-                                        right = Dual{T, V, P}(sol.right, partials))
+            sol.resid; retcode = sol.retcode,
+            left = Dual{T, V, P}(sol.left, partials),
+            right = Dual{T, V, P}(sol.right, partials))
         #return BracketingSolution(Dual{T,V,P}(sol.left, partials), Dual{T,V,P}(sol.right, partials), sol.retcode, sol.resid)
     end
     @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
-                                                                  <:AbstractArray{
-                                                                                  <:Dual{T,
-                                                                                         V,
-                                                                                         P}
-                                                                                  }},
-                                   alg::$Alg, args...;
-                                   kwargs...) where {uType, iip, T, V, P}
+            <:AbstractArray{
+                <:Dual{T,
+                    V,
+                    P},
+            }},
+        alg::$Alg, args...;
+        kwargs...) where {uType, iip, T, V, P}
         sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
         return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials),
-                                        sol.resid; retcode = sol.retcode,
-                                        left = Dual{T, V, P}(sol.left, partials),
-                                        right = Dual{T, V, P}(sol.right, partials))
+            sol.resid; retcode = sol.retcode,
+            left = Dual{T, V, P}(sol.left, partials),
+            right = Dual{T, V, P}(sol.right, partials))
         #return BracketingSolution(Dual{T,V,P}(sol.left, partials), Dual{T,V,P}(sol.right, partials), sol.retcode, sol.resid)
     end
 end
diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 61861a4a7..a2669ca96 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -9,9 +9,9 @@ algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal
 struct Alefeld <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem,
-                         alg::Alefeld, args...; abstol = nothing,
-                         reltol = nothing,
-                         maxiters = 1000, kwargs...)
+    alg::Alefeld, args...; abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     c = a - (b - a) / (f(b) - f(a)) * f(a)
@@ -19,14 +19,14 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
     fc = f(c)
     (a == c || b == c) &&
         return SciMLBase.build_solution(prob, alg, c, fc;
-                                        retcode = ReturnCode.FloatingPointLimit,
-                                        left = a,
-                                        right = b)
+            retcode = ReturnCode.FloatingPointLimit,
+            left = a,
+            right = b)
     iszero(fc) &&
         return SciMLBase.build_solution(prob, alg, c, fc;
-                                        retcode = ReturnCode.Success,
-                                        left = a,
-                                        right = b)
+            retcode = ReturnCode.Success,
+            left = a,
+            right = b)
     a, b, d = _bracket(f, a, b, c)
     e = zero(a)   # Set e as 0 before iteration to avoid a non-value f(e)
 
@@ -45,14 +45,14 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
         ē, fc = d, f(c)
         (a == c || b == c) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
-                                            retcode = ReturnCode.FloatingPointLimit,
-                                            left = a,
-                                            right = b)
+                retcode = ReturnCode.FloatingPointLimit,
+                left = a,
+                right = b)
         iszero(fc) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
-                                            retcode = ReturnCode.Success,
-                                            left = a,
-                                            right = b)
+                retcode = ReturnCode.Success,
+                left = a,
+                right = b)
         ā, b̄, d̄ = _bracket(f, a, b, c)
 
         # The second bracketing block
@@ -68,14 +68,14 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
         fc = f(c)
         (ā == c || b̄ == c) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
-                                            retcode = ReturnCode.FloatingPointLimit,
-                                            left = ā,
-                                            right = b̄)
+                retcode = ReturnCode.FloatingPointLimit,
+                left = ā,
+                right = b̄)
         iszero(fc) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
-                                            retcode = ReturnCode.Success,
-                                            left = ā,
-                                            right = b̄)
+                retcode = ReturnCode.Success,
+                left = ā,
+                right = b̄)
         ā, b̄, d̄ = _bracket(f, ā, b̄, c)
 
         # The third bracketing block
@@ -91,14 +91,14 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
         fc = f(c)
         (ā == c || b̄ == c) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
-                                            retcode = ReturnCode.FloatingPointLimit,
-                                            left = ā,
-                                            right = b̄)
+                retcode = ReturnCode.FloatingPointLimit,
+                left = ā,
+                right = b̄)
         iszero(fc) &&
             return SciMLBase.build_solution(prob, alg, c, fc;
-                                            retcode = ReturnCode.Success,
-                                            left = ā,
-                                            right = b̄)
+                retcode = ReturnCode.Success,
+                left = ā,
+                right = b̄)
         ā, b̄, d = _bracket(f, ā, b̄, c)
 
         # The last bracketing block
@@ -110,14 +110,14 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
             fc = f(c)
             (ā == c || b̄ == c) &&
                 return SciMLBase.build_solution(prob, alg, c, fc;
-                                                retcode = ReturnCode.FloatingPointLimit,
-                                                left = ā,
-                                                right = b̄)
+                    retcode = ReturnCode.FloatingPointLimit,
+                    left = ā,
+                    right = b̄)
             iszero(fc) &&
                 return SciMLBase.build_solution(prob, alg, c, fc;
-                                                retcode = ReturnCode.Success,
-                                                left = ā,
-                                                right = b̄)
+                    retcode = ReturnCode.Success,
+                    left = ā,
+                    right = b̄)
             a, b, d = _bracket(f, ā, b̄, c)
         end
     end
@@ -132,7 +132,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
 
     # Reuturn solution when run out of max interation
     return SciMLBase.build_solution(prob, alg, c, fc; retcode = ReturnCode.MaxIters,
-                                    left = a, right = b)
+        left = a, right = b)
 end
 
 # Define subrotine function bracket, check fc before bracket to return solution
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 4e2b232e6..673f77067 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -20,16 +20,16 @@ function Bisection(; exact_left = false, exact_right = false)
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...;
-                         maxiters = 1000,
-                         kwargs...)
+    maxiters = 1000,
+    kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
-                                        retcode = ReturnCode.ExactSolutionLeft, left = left,
-                                        right = right)
+            retcode = ReturnCode.ExactSolutionLeft, left = left,
+            right = right)
     end
 
     i = 1
@@ -38,8 +38,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
             mid = (left + right) / 2
             (mid == left || mid == right) &&
                 return SciMLBase.build_solution(prob, alg, left, fl;
-                                                retcode = ReturnCode.FloatingPointLimit,
-                                                left = left, right = right)
+                    retcode = ReturnCode.FloatingPointLimit,
+                    left = left, right = right)
             fm = f(mid)
             if iszero(fm)
                 right = mid
@@ -60,8 +60,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
         mid = (left + right) / 2
         (mid == left || mid == right) &&
             return SciMLBase.build_solution(prob, alg, left, fl;
-                                            retcode = ReturnCode.FloatingPointLimit,
-                                            left = left, right = right)
+                retcode = ReturnCode.FloatingPointLimit,
+                left = left, right = right)
         fm = f(mid)
         if iszero(fm)
             right = mid
@@ -74,5 +74,5 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
     end
 
     return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
-                                    left = left, right = right)
+        left = left, right = right)
 end
diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index 99f645f6a..1cedad134 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -7,8 +7,8 @@ A non-allocating Brent method
 struct Brent <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
-                         maxiters = 1000,
-                         kwargs...)
+    maxiters = 1000,
+    kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     fa, fb = f(a), f(b)
@@ -16,8 +16,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
 
     if iszero(fa)
         return SciMLBase.build_solution(prob, alg, a, fa;
-                                        retcode = ReturnCode.ExactSolutionLeft, left = a,
-                                        right = b)
+            retcode = ReturnCode.ExactSolutionLeft, left = a,
+            right = b)
     end
     if abs(fa) < abs(fb)
         c = b
@@ -53,8 +53,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
                 s = (a + b) / 2
                 (s == a || s == b) &&
                     return SciMLBase.build_solution(prob, alg, a, fa;
-                                                    retcode = ReturnCode.FloatingPointLimit,
-                                                    left = a, right = b)
+                        retcode = ReturnCode.FloatingPointLimit,
+                        left = a, right = b)
                 cond = true
             else
                 cond = false
@@ -95,8 +95,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
         c = (a + b) / 2
         if (c == a || c == b)
             return SciMLBase.build_solution(prob, alg, a, fa;
-                                            retcode = ReturnCode.FloatingPointLimit,
-                                            left = a, right = b)
+                retcode = ReturnCode.FloatingPointLimit,
+                left = a, right = b)
         end
         fc = f(c)
         if iszero(fc)
@@ -110,5 +110,5 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
     end
 
     return SciMLBase.build_solution(prob, alg, a, fa; retcode = ReturnCode.MaxIters,
-                                    left = a, right = b)
+        left = a, right = b)
 end
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index e46469619..8ce0d66b5 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -16,15 +16,15 @@ struct Broyden{batched, TC <: NLSolveTerminationCondition} <:
     termination_condition::TC
 
     function Broyden(; batched = false,
-                     termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-                                                                         abstol = nothing,
-                                                                         reltol = nothing))
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing))
         return new{batched, typeof(termination_condition)}(termination_condition)
     end
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{false}, args...;
-                           abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
+    abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
     f = Base.Fix2(prob.f, prob.p)
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index 3d2e9c8aa..e898f060b 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -52,15 +52,15 @@ struct SimpleDFSane{T} <: AbstractSimpleNonlinearSolveAlgorithm
     η_strategy::Function
 
     function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
-                          M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
-                          nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 / k^2)
+        M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 / k^2)
         new{typeof(σ_min)}(σ_min, σ_max, σ_1, M, γ, τ_min, τ_max, nexp, η_strategy)
     end
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
-                           args...; abstol = nothing, reltol = nothing, maxiters = 1000,
-                           kwargs...)
+    args...; abstol = nothing, reltol = nothing, maxiters = 1000,
+    kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     T = eltype(x)
@@ -96,7 +96,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
     for k in 1:maxiters
         iszero(F_k) &&
             return SciMLBase.build_solution(prob, alg, x, F_k;
-                                            retcode = ReturnCode.Success)
+                retcode = ReturnCode.Success)
 
         # Spectral parameter range check
         if abs(σ_k) > σ_max
@@ -136,7 +136,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
 
         if isapprox(x_new, x, atol = atol, rtol = rtol)
             return SciMLBase.build_solution(prob, alg, x_new, F_new;
-                                            retcode = ReturnCode.Success)
+                retcode = ReturnCode.Success)
         end
         # Update spectral parameter
         s_k = x_new - x
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index 81ce68ffa..cce11a811 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -4,16 +4,16 @@
 struct Falsi <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
-                         maxiters = 1000,
-                         kwargs...)
+    maxiters = 1000,
+    kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
-                                        retcode = ReturnCode.ExactSolutionLeft, left = left,
-                                        right = right)
+            retcode = ReturnCode.ExactSolutionLeft, left = left,
+            right = right)
     end
 
     i = 1
@@ -21,8 +21,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
         while i < maxiters
             if nextfloat_tdir(left, prob.tspan...) == right
                 return SciMLBase.build_solution(prob, alg, left, fl;
-                                                retcode = ReturnCode.FloatingPointLimit,
-                                                left = left, right = right)
+                    retcode = ReturnCode.FloatingPointLimit,
+                    left = left, right = right)
             end
             mid = (fr * left - fl * right) / (fr - fl)
             for i in 1:10
@@ -51,8 +51,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
         mid = (left + right) / 2
         (mid == left || mid == right) &&
             return SciMLBase.build_solution(prob, alg, left, fl;
-                                            retcode = ReturnCode.FloatingPointLimit,
-                                            left = left, right = right)
+                retcode = ReturnCode.FloatingPointLimit,
+                left = left, right = right)
         fm = f(mid)
         if iszero(fm)
             right = mid
@@ -68,5 +68,5 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
     end
 
     return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
-                                    left = left, right = right)
+        left = left, right = right)
 end
diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index cdda7beda..9e97a57cd 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -30,16 +30,16 @@ and static array problems.
 """
 struct Halley{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     function Halley(; chunk_size = Val{0}(), autodiff = Val{true}(),
-                    diff_type = Val{:forward})
+        diff_type = Val{:forward})
         new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
             SciMLBase._unwrap_val(diff_type)}()
     end
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-                           alg::Halley, args...; abstol = nothing,
-                           reltol = nothing,
-                           maxiters = 1000, kwargs...)
+    alg::Halley, args...; abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fx = f(x)
@@ -81,18 +81,18 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             if isa(x, Number)
                 fx = f(x)
                 dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg),
-                                                              eltype(x))
+                    eltype(x))
                 d2fx = FiniteDiff.finite_difference_derivative(x -> FiniteDiff.finite_difference_derivative(f,
-                                                                                                            x),
-                                                               x,
-                                                               diff_type(alg), eltype(x))
+                        x),
+                    x,
+                    diff_type(alg), eltype(x))
             else
                 fx = f(x)
                 dfx = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x))
                 d2fx = FiniteDiff.finite_difference_jacobian(x -> FiniteDiff.finite_difference_jacobian(f,
-                                                                                                        x),
-                                                             x,
-                                                             diff_type(alg), eltype(x))
+                        x),
+                    x,
+                    diff_type(alg), eltype(x))
                 ai = -(dfx \ fx)
                 A = reshape(d2fx * ai, (n, n))
                 bi = (dfx) \ (A * ai)
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 4d59377df..00264d32f 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -9,9 +9,9 @@ This method is non-allocating on scalar problems.
 struct Klement <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-                           alg::Klement, args...; abstol = nothing,
-                           reltol = nothing,
-                           maxiters = 1000, kwargs...)
+    alg::Klement, args...; abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fₙ = f(x)
@@ -39,11 +39,11 @@ function SciMLBase.__solve(prob::NonlinearProblem,
 
             iszero(fₙ) &&
                 return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                                retcode = ReturnCode.Success)
+                    retcode = ReturnCode.Success)
 
             if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
                 return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                                retcode = ReturnCode.Success)
+                    retcode = ReturnCode.Success)
             end
 
             Δxₙ = xₙ - xₙ₋₁
@@ -81,11 +81,11 @@ function SciMLBase.__solve(prob::NonlinearProblem,
 
             iszero(fₙ) &&
                 return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                                retcode = ReturnCode.Success)
+                    retcode = ReturnCode.Success)
 
             if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
                 return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                                                retcode = ReturnCode.Success)
+                    retcode = ReturnCode.Success)
             end
 
             Δxₙ = xₙ - xₙ₋₁
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
index 6cca7c11e..fc2b51a88 100644
--- a/lib/SimpleNonlinearSolve/src/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -18,16 +18,16 @@ struct LBroyden{batched, TC <: NLSolveTerminationCondition} <:
     threshold::Int
 
     function LBroyden(; batched = false, threshold::Int = 27,
-                      termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-                                                                          abstol = nothing,
-                                                                          reltol = nothing))
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing))
         return new{batched, typeof(termination_condition)}(termination_condition, threshold)
     end
 end
 
 @views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden{batched}, args...;
-                                  abstol = nothing, reltol = nothing, maxiters = 1000,
-                                  kwargs...) where {batched}
+    abstol = nothing, reltol = nothing, maxiters = 1000,
+    kwargs...) where {batched}
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
     threshold = min(maxiters, alg.threshold)
@@ -93,7 +93,7 @@ end
         selectdim(U, 1, mod1(i, threshold)) .= u
 
         update = -_matvec(selectdim(U, 1, 1:min(threshold, i + 1)),
-                          selectdim(Vᵀ, 2, 1:min(threshold, i + 1)), fₙ)
+            selectdim(Vᵀ, 2, 1:min(threshold, i + 1)), fₙ)
 
         xₙ₋₁ = xₙ
         fₙ₋₁ = fₙ
@@ -116,26 +116,26 @@ function _init_lbroyden_state(batched::Bool, x, threshold)
 end
 
 function _rmatvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
-                  x::Union{<:AbstractVector, <:Number})
+    x::Union{<:AbstractVector, <:Number})
     length(U) == 0 && return x
     return -x .+ vec((x' * Vᵀ) * U)
 end
 
 function _rmatvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
-                  x::AbstractMatrix) where {T1, T2}
+    x::AbstractMatrix) where {T1, T2}
     length(U) == 0 && return x
     Vᵀx = sum(Vᵀ .* reshape(x, size(x, 1), 1, size(x, 2)); dims = 1)
     return -x .+ _drdims_sum(U .* permutedims(Vᵀx, (2, 1, 3)); dims = 1)
 end
 
 function _matvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
-                 x::Union{<:AbstractVector, <:Number})
+    x::Union{<:AbstractVector, <:Number})
     length(U) == 0 && return x
     return -x .+ vec(Vᵀ * (U * x))
 end
 
 function _matvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
-                 x::AbstractMatrix) where {T1, T2}
+    x::AbstractMatrix) where {T1, T2}
     length(U) == 0 && return x
     xUᵀ = sum(reshape(x, size(x, 1), 1, size(x, 2)) .* permutedims(U, (2, 1, 3)); dims = 1)
     return -x .+ _drdims_sum(xUᵀ .* Vᵀ; dims = 2)
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 8d2d9796c..386c35053 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -30,16 +30,16 @@ and static array problems.
 """
 struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     function SimpleNewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
-                                 diff_type = Val{:forward})
+        diff_type = Val{:forward})
         new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
             SciMLBase._unwrap_val(diff_type)}()
     end
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-                           alg::SimpleNewtonRaphson, args...; abstol = nothing,
-                           reltol = nothing,
-                           maxiters = 1000, kwargs...)
+    alg::SimpleNewtonRaphson, args...; abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fx = float(prob.u0)
@@ -71,7 +71,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         else
             fx = f(x)
             dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x),
-                                                          fx)
+                fx)
         end
         iszero(fx) &&
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index 9fe7bccfe..62b5a931a 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -7,16 +7,16 @@ A non-allocating ridder method
 struct Ridder <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
-                         maxiters = 1000,
-                         kwargs...)
+    maxiters = 1000,
+    kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
-                                        retcode = ReturnCode.ExactSolutionLeft, left = left,
-                                        right = right)
+            retcode = ReturnCode.ExactSolutionLeft, left = left,
+            right = right)
     end
 
     xo = oftype(left, Inf)
@@ -26,14 +26,14 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
             mid = (left + right) / 2
             (mid == left || mid == right) &&
                 return SciMLBase.build_solution(prob, alg, left, fl;
-                                                retcode = ReturnCode.FloatingPointLimit,
-                                                left = left, right = right)
+                    retcode = ReturnCode.FloatingPointLimit,
+                    left = left, right = right)
             fm = f(mid)
             s = sqrt(fm^2 - fl * fr)
             iszero(s) &&
                 return SciMLBase.build_solution(prob, alg, left, fl;
-                                                retcode = ReturnCode.Failure,
-                                                left = left, right = right)
+                    retcode = ReturnCode.Failure,
+                    left = left, right = right)
             x = mid + (mid - left) * sign(fl - fr) * fm / s
             fx = f(x)
             xo = x
@@ -63,8 +63,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
         mid = (left + right) / 2
         (mid == left || mid == right) &&
             return SciMLBase.build_solution(prob, alg, left, fl;
-                                            retcode = ReturnCode.FloatingPointLimit,
-                                            left = left, right = right)
+                retcode = ReturnCode.FloatingPointLimit,
+                left = left, right = right)
         fm = f(mid)
         if iszero(fm)
             right = mid
@@ -77,5 +77,5 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
     end
 
     return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
-                                    left = left, right = right)
+        left = left, right = right)
 end
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index aa893ffc1..1423d59b8 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -64,34 +64,34 @@ struct SimpleTrustRegion{T, CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     expand_factor::T
     max_shrink_times::Int
     function SimpleTrustRegion(; chunk_size = Val{0}(),
-                               autodiff = Val{true}(),
-                               diff_type = Val{:forward},
-                               max_trust_radius::Real = 0.0,
-                               initial_trust_radius::Real = 0.0,
-                               step_threshold::Real = 0.1,
-                               shrink_threshold::Real = 0.25,
-                               expand_threshold::Real = 0.75,
-                               shrink_factor::Real = 0.25,
-                               expand_factor::Real = 2.0,
-                               max_shrink_times::Int = 32)
+        autodiff = Val{true}(),
+        diff_type = Val{:forward},
+        max_trust_radius::Real = 0.0,
+        initial_trust_radius::Real = 0.0,
+        step_threshold::Real = 0.1,
+        shrink_threshold::Real = 0.25,
+        expand_threshold::Real = 0.75,
+        shrink_factor::Real = 0.25,
+        expand_factor::Real = 2.0,
+        max_shrink_times::Int = 32)
         new{typeof(initial_trust_radius),
             SciMLBase._unwrap_val(chunk_size),
             SciMLBase._unwrap_val(autodiff),
             SciMLBase._unwrap_val(diff_type)}(max_trust_radius,
-                                              initial_trust_radius,
-                                              step_threshold,
-                                              shrink_threshold,
-                                              expand_threshold,
-                                              shrink_factor,
-                                              expand_factor,
-                                              max_shrink_times)
+            initial_trust_radius,
+            step_threshold,
+            shrink_threshold,
+            expand_threshold,
+            shrink_factor,
+            expand_factor,
+            max_shrink_times)
     end
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-                           alg::SimpleTrustRegion, args...; abstol = nothing,
-                           reltol = nothing,
-                           maxiters = 1000, kwargs...)
+    alg::SimpleTrustRegion, args...; abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     T = typeof(x)
@@ -112,7 +112,6 @@ function SciMLBase.__solve(prob::NonlinearProblem,
            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
     rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
 
-
     if DiffEqBase.has_jac(prob.f)
         ∇f = prob.f.jac(x, prob.p)
         F = f(x)
@@ -156,7 +155,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             shrink_counter += 1
             if shrink_counter > max_shrink_times
                 return SciMLBase.build_solution(prob, alg, x, F;
-                                                retcode = ReturnCode.Success)
+                    retcode = ReturnCode.Success)
             end
         else
             shrink_counter = 0
@@ -164,7 +163,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         if r > η₁
             if isapprox(xₖ₊₁, x, atol = atol, rtol = rtol)
                 return SciMLBase.build_solution(prob, alg, xₖ₊₁, Fₖ₊₁;
-                                                retcode = ReturnCode.Success)
+                    retcode = ReturnCode.Success)
             end
             # Take the step.
             x = xₖ₊₁
@@ -173,16 +172,16 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                 F, ∇f = value_derivative(f, x)
             elseif x isa AbstractArray
                 ∇f = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x),
-                                                           F)
+                    F)
             else
                 ∇f = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg),
-                                                             eltype(x),
-                                                             F)
+                    eltype(x),
+                    F)
             end
 
             iszero(F) &&
                 return SciMLBase.build_solution(prob, alg, x, F;
-                                                retcode = ReturnCode.Success)
+                    retcode = ReturnCode.Success)
 
             # Update the trust region radius.
             if r > η₃ && norm(δ) ≈ Δ
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 1a0249b65..e1d6f9bf8 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -18,7 +18,7 @@ for mode in instances(NLSolveTerminationMode.T)
     end
 
     termination_condition = NLSolveTerminationCondition(mode; abstol = nothing,
-                                                        reltol = nothing)
+        reltol = nothing)
     push!(BROYDEN_SOLVERS, Broyden(; batched = false, termination_condition))
     push!(BATCHED_BROYDEN_SOLVERS, Broyden(; batched = true, termination_condition))
     push!(LBROYDEN_SOLVERS, LBroyden(; batched = false, termination_condition))
@@ -131,7 +131,7 @@ using ForwardDiff
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
 for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane(), Halley(), BROYDEN_SOLVERS...)
+    SimpleDFSane(), Halley(), BROYDEN_SOLVERS...)
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, abstol = 1e-9)
@@ -154,7 +154,7 @@ end
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
 for alg in (SimpleNewtonRaphson(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane(), Halley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
+    SimpleDFSane(), Halley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -251,7 +251,7 @@ for alg in [Bisection(), Falsi(), Ridder(), Brent()]
 end
 
 for alg in (SimpleNewtonRaphson(), Klement(), SimpleTrustRegion(),
-            SimpleDFSane(), Halley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
+    SimpleDFSane(), Halley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -267,10 +267,10 @@ f, u0 = (u, p) -> u .* u .- 2.0, @SVector[1.0, 1.0]
 probN = NonlinearProblem(f, u0)
 
 for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
-            SimpleTrustRegion(),
-            SimpleTrustRegion(; autodiff = false), Halley(), Halley(; autodiff = false),
-            Klement(), SimpleDFSane(),
-            BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
+    SimpleTrustRegion(),
+    SimpleTrustRegion(; autodiff = false), Halley(), Halley(; autodiff = false),
+    Klement(), SimpleDFSane(),
+    BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
     sol = solve(probN, alg)
 
     @test sol.retcode == ReturnCode.Success
@@ -284,8 +284,8 @@ for u0 in [1.0, [1, 1.0]]
     sol = sqrt(2) * u0
 
     for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
-                SimpleTrustRegion(), SimpleTrustRegion(; autodiff = false), Klement(),
-                SimpleDFSane(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
+        SimpleTrustRegion(), SimpleTrustRegion(; autodiff = false), Klement(),
+        SimpleDFSane(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
         sol2 = solve(probN, alg)
 
         @test sol2.retcode == ReturnCode.Success
@@ -399,18 +399,18 @@ expand_factor = [1.5, 2.0, 3.0]
 max_shrink_times = [10, 20, 30]
 
 list_of_options = zip(max_trust_radius, initial_trust_radius, step_threshold,
-                      shrink_threshold, expand_threshold, shrink_factor,
-                      expand_factor, max_shrink_times)
+    shrink_threshold, expand_threshold, shrink_factor,
+    expand_factor, max_shrink_times)
 for options in list_of_options
     local probN, sol, alg
     alg = SimpleTrustRegion(max_trust_radius = options[1],
-                            initial_trust_radius = options[2],
-                            step_threshold = options[3],
-                            shrink_threshold = options[4],
-                            expand_threshold = options[5],
-                            shrink_factor = options[6],
-                            expand_factor = options[7],
-                            max_shrink_times = options[8])
+        initial_trust_radius = options[2],
+        step_threshold = options[3],
+        shrink_threshold = options[4],
+        expand_threshold = options[5],
+        shrink_factor = options[6],
+        expand_factor = options[7],
+        max_shrink_times = options[8])
 
     probN = NonlinearProblem(f, u0, p)
     sol = solve(probN, alg)
@@ -454,18 +454,18 @@ nexp = [2, 1, 2]
 ]
 
 list_of_options = zip(σ_min, σ_max, σ_1, M, γ, τ_min, τ_max, nexp,
-                      η_strategy)
+    η_strategy)
 for options in list_of_options
     local probN, sol, alg
     alg = SimpleDFSane(σ_min = options[1],
-                       σ_max = options[2],
-                       σ_1 = options[3],
-                       M = options[4],
-                       γ = options[5],
-                       τ_min = options[6],
-                       τ_max = options[7],
-                       nexp = options[8],
-                       η_strategy = options[9])
+        σ_max = options[2],
+        σ_1 = options[3],
+        M = options[4],
+        γ = options[5],
+        τ_min = options[6],
+        τ_max = options[7],
+        nexp = options[8],
+        η_strategy = options[9])
 
     probN = NonlinearProblem(f, u0, p)
     sol = solve(probN, alg)
@@ -495,7 +495,9 @@ end
 
 f, u0 = (u, p) -> u .* u .- p, randn(3)
 
-f_jac(u, p) = begin diagm(2 * u) end
+f_jac(u, p) = begin
+    diagm(2 * u)
+end
 
 p = [2.0, 1.0, 5.0];
 
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 7269ed6bf..94a0086e4 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -6,7 +6,9 @@ const GROUP = get(ENV, "GROUP", "All")
 const is_APPVEYOR = Sys.iswindows() && haskey(ENV, "APPVEYOR")
 
 @time begin
-
-if GROUP == "All" || GROUP == "Core"
-    @time @safetestset "Basic Tests + Some AD" begin include("basictests.jl") end
-end end
+    if GROUP == "All" || GROUP == "Core"
+        @time @safetestset "Basic Tests + Some AD" begin
+            include("basictests.jl")
+        end
+    end
+end

From 31820602e7051f74232dbbe15120b858ba75419b Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 8 Jun 2023 17:17:23 -0400
Subject: [PATCH 122/375] Batched DFSane

---
 lib/SimpleNonlinearSolve/Project.toml       |   2 +-
 lib/SimpleNonlinearSolve/src/dfsane.jl      | 174 +++++++++++++++-----
 lib/SimpleNonlinearSolve/test/basictests.jl |  12 +-
 3 files changed, 142 insertions(+), 46 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index c1f21c72e..cf5a675af 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.15"
+version = "0.1.16"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index e898f060b..388ebfe89 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -1,9 +1,12 @@
 """
-```julia
-SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
-             M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
-             nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 / k^2)
-```
+    SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
+        M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2,
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing),
+        batched::Bool = false,
+        max_inner_iterations::Int = 1000)
 
 A low-overhead implementation of the df-sane method for solving large-scale nonlinear
 systems of equations. For in depth information about all the parameters and the algorithm,
@@ -39,8 +42,16 @@ Computation, 75, 1429-1448.](https://www.researchgate.net/publication/220576479_
   ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and
   `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to
   ``||F||^2 / k^2``.
+- `termination_condition`: a `NLSolveTerminationCondition` that determines when the solver
+  should terminate. Defaults to `NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+  abstol = nothing, reltol = nothing)`.
+- `batched`: if `true`, the algorithm will use a batched version of the algorithm that treats each
+  column of `x` as a separate problem. This can be useful nonlinear problems involing neural
+  networks. Defaults to `false`.
+- `max_inner_iterations`: the maximum number of iterations allowed for the inner loop of the
+  algorithm. Used exclusively in `batched` mode. Defaults to `1000`.
 """
-struct SimpleDFSane{T} <: AbstractSimpleNonlinearSolveAlgorithm
+struct SimpleDFSane{batched, T, TC} <: AbstractSimpleNonlinearSolveAlgorithm
     σ_min::T
     σ_max::T
     σ_1::T
@@ -50,23 +61,49 @@ struct SimpleDFSane{T} <: AbstractSimpleNonlinearSolveAlgorithm
     τ_max::T
     nexp::Int
     η_strategy::Function
+    termination_condition::TC
+    max_inner_iterations::Int
 
     function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
         M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
-        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 / k^2)
-        new{typeof(σ_min)}(σ_min, σ_max, σ_1, M, γ, τ_min, τ_max, nexp, η_strategy)
+        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2,
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing),
+        batched::Bool = false,
+        max_inner_iterations = 1000)
+        return new{batched, typeof(σ_min), typeof(termination_condition)}(σ_min,
+            σ_max,
+            σ_1,
+            M,
+            γ,
+            τ_min,
+            τ_max,
+            nexp,
+            η_strategy,
+            termination_condition,
+            max_inner_iterations)
     end
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
     args...; abstol = nothing, reltol = nothing, maxiters = 1000,
-    kwargs...)
+    kwargs...) where {batched}
+    tc = alg.termination_condition
+    mode = DiffEqBase.get_termination_mode(tc)
+
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
+
+    if batched
+        batch_size = size(x, 2)
+    end
+
     T = eltype(x)
     σ_min = float(alg.σ_min)
     σ_max = float(alg.σ_max)
-    σ_k = float(alg.σ_1)
+    σ_k = batched ? fill(float(alg.σ_1), 1, batch_size) : float(alg.σ_1)
+
     M = alg.M
     γ = float(alg.γ)
     τ_min = float(alg.τ_min)
@@ -74,74 +111,123 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
     nexp = alg.nexp
     η_strategy = alg.η_strategy
 
+    batched && @assert ndims(x)==2 "Batched SimpleDFSane only supports 2D arrays"
+
     if SciMLBase.isinplace(prob)
         error("SimpleDFSane currently only supports out-of-place nonlinear problems")
     end
 
     atol = abstol !== nothing ? abstol :
-           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
-    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+           (tc.abstol !== nothing ? tc.abstol :
+            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
+    rtol = reltol !== nothing ? reltol :
+           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
+
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        error("SimpleDFSane currently doesn't support SAFE_BEST termination modes")
+    end
+
+    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
+              nothing
+    termination_condition = tc(storage)
 
     function ff(x)
         F = f(x)
-        f_k = norm(F)^nexp
+        f_k = if batched
+            sum(abs2, F; dims = 1) .^ (nexp / 2)
+        else
+            norm(F)^nexp
+        end
         return f_k, F
     end
 
+    function generate_history(f_k, M)
+        if batched
+            history = similar(f_k, (M, length(f_k)))
+            history .= reshape(f_k, 1, :)
+            return history
+        else
+            return fill(f_k, M)
+        end
+    end
+
     f_k, F_k = ff(x)
     α_1 = convert(T, 1.0)
     f_1 = f_k
-    history_f_k = fill(f_k, M)
+    history_f_k = generate_history(f_k, M)
 
     for k in 1:maxiters
-        iszero(F_k) &&
-            return SciMLBase.build_solution(prob, alg, x, F_k;
-                retcode = ReturnCode.Success)
-
         # Spectral parameter range check
-        if abs(σ_k) > σ_max
-            σ_k = sign(σ_k) * σ_max
-        elseif abs(σ_k) < σ_min
-            σ_k = sign(σ_k) * σ_min
+        if batched
+            @. σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
+        else
+            σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
         end
 
         # Line search direction
-        d = -σ_k * F_k
+        d = -σ_k .* F_k
 
         η = η_strategy(f_1, k, x, F_k)
-        f̄ = maximum(history_f_k)
+        f̄ = batched ? maximum(history_f_k; dims = 1) : maximum(history_f_k)
         α_p = α_1
         α_m = α_1
-        x_new = x + α_p * d
+        x_new = @. x + α_p * d
+
         f_new, F_new = ff(x_new)
+
+        inner_iterations = 0
         while true
-            if f_new ≤ f̄ + η - γ * α_p^2 * f_k
-                break
+            inner_iterations += 1
+
+            if batched
+                # NOTE: This is simply a heuristic, ideally we check using `all` but that is
+                #       typically very expensive for large problems
+                norm(f_new) ≤ norm(@. f̄ + η - γ * α_p^2 * f_k) && break
+            else
+                f_new ≤ f̄ + η - γ * α_p^2 * f_k && break
             end
 
-            α_tp = α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
-            x_new = x - α_m * d
+            α_tp = @. α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
+            x_new = @. x - α_m * d
             f_new, F_new = ff(x_new)
 
-            if f_new ≤ f̄ + η - γ * α_m^2 * f_k
-                break
+            if batched
+                # NOTE: This is simply a heuristic, ideally we check using `all` but that is
+                #       typically very expensive for large problems
+                norm(f_new) ≤ norm(@. f̄ + η - γ * α_p^2 * f_k) && break
+            else
+                f_new ≤ f̄ + η - γ * α_p^2 * f_k && break
             end
 
-            α_tm = α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
-            α_p = min(τ_max * α_p, max(α_tp, τ_min * α_p))
-            α_m = min(τ_max * α_m, max(α_tm, τ_min * α_m))
-            x_new = x + α_p * d
+            α_tm = @. α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
+            α_p = @. clamp(α_tp, τ_min * α_p, τ_max * α_p)
+            α_m = @. clamp(α_tm, τ_min * α_m, τ_max * α_m)
+            x_new = @. x + α_p * d
             f_new, F_new = ff(x_new)
+
+            # NOTE: The original algorithm runs till either condition is satisfied, however,
+            #       for most batched problems like neural networks we only care about
+            #       approximate convergence
+            batched && (inner_iterations ≥ alg.max_inner_iterations) && break
         end
 
-        if isapprox(x_new, x, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, x_new, F_new;
+        if termination_condition(F_new, x_new, x, atol, rtol)
+            return SciMLBase.build_solution(prob,
+                alg,
+                x_new,
+                F_new;
                 retcode = ReturnCode.Success)
         end
+
         # Update spectral parameter
-        s_k = x_new - x
-        y_k = F_new - F_k
-        σ_k = (s_k' * s_k) / (s_k' * y_k)
+        s_k = @. x_new - x
+        y_k = @. F_new - F_k
+
+        if batched
+            σ_k = sum(abs2, s_k; dims = 1) ./ (sum(s_k .* y_k; dims = 1) .+ T(1e-5))
+        else
+            σ_k = (s_k' * s_k) / (s_k' * y_k)
+        end
 
         # Take step
         x = x_new
@@ -149,7 +235,11 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
         f_k = f_new
 
         # Store function value
-        history_f_k[k % M + 1] = f_new
+        if batched
+            history_f_k[k % M + 1, :] .= vec(f_new)
+        else
+            history_f_k[k % M + 1] = f_new
+        end
     end
     return SciMLBase.build_solution(prob, alg, x, F_k; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index e1d6f9bf8..aea173157 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -9,6 +9,8 @@ const BATCHED_BROYDEN_SOLVERS = Broyden[]
 const BROYDEN_SOLVERS = Broyden[]
 const BATCHED_LBROYDEN_SOLVERS = LBroyden[]
 const LBROYDEN_SOLVERS = LBroyden[]
+const BATCHED_DFSANE_SOLVERS = SimpleDFSane[]
+const DFSANE_SOLVERS = SimpleDFSane[]
 
 for mode in instances(NLSolveTerminationMode.T)
     if mode ∈
@@ -23,6 +25,8 @@ for mode in instances(NLSolveTerminationMode.T)
     push!(BATCHED_BROYDEN_SOLVERS, Broyden(; batched = true, termination_condition))
     push!(LBROYDEN_SOLVERS, LBroyden(; batched = false, termination_condition))
     push!(BATCHED_LBROYDEN_SOLVERS, LBroyden(; batched = true, termination_condition))
+    push!(DFSANE_SOLVERS, SimpleDFSane(; batched = false, termination_condition))
+    push!(BATCHED_DFSANE_SOLVERS, SimpleDFSane(; batched = true, termination_condition))
 end
 
 # SimpleNewtonRaphson
@@ -484,11 +488,13 @@ sol = solve(probN, Broyden(batched = true))
 
 @test abs.(sol.u) ≈ sqrt.(p)
 
-for alg in (BATCHED_BROYDEN_SOLVERS..., BATCHED_LBROYDEN_SOLVERS...)
-    sol = solve(probN, alg)
+for alg in (BATCHED_BROYDEN_SOLVERS...,
+    BATCHED_LBROYDEN_SOLVERS...,
+    BATCHED_DFSANE_SOLVERS...)
+    sol = solve(probN, alg; abstol = 1e-3, reltol = 1e-3)
 
     @test sol.retcode == ReturnCode.Success
-    @test abs.(sol.u) ≈ sqrt.(p)
+    @test abs.(sol.u)≈sqrt.(p) atol=1e-3 rtol=1e-3
 end
 
 ## User specified Jacobian

From 044b7431ea07b8beb79808e65eb455636593ef0b Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 21 Jun 2023 12:40:57 -0400
Subject: [PATCH 123/375] Reuse criteria computation

---
 lib/SimpleNonlinearSolve/src/dfsane.jl | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index 388ebfe89..b5e2b8200 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -180,11 +180,13 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
             inner_iterations += 1
 
             if batched
+                criteria = @. f̄ + η - γ * α_p^2 * f_k
                 # NOTE: This is simply a heuristic, ideally we check using `all` but that is
                 #       typically very expensive for large problems
-                norm(f_new) ≤ norm(@. f̄ + η - γ * α_p^2 * f_k) && break
+                (sum(f_new .≤ criteria) ≥ batch_size ÷ 2) && break
             else
-                f_new ≤ f̄ + η - γ * α_p^2 * f_k && break
+                criteria = f̄ + η - γ * α_p^2 * f_k
+                f_new ≤ criteria && break
             end
 
             α_tp = @. α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
@@ -194,9 +196,9 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
             if batched
                 # NOTE: This is simply a heuristic, ideally we check using `all` but that is
                 #       typically very expensive for large problems
-                norm(f_new) ≤ norm(@. f̄ + η - γ * α_p^2 * f_k) && break
+                (sum(f_new .≤ criteria) ≥ batch_size ÷ 2) && break
             else
-                f_new ≤ f̄ + η - γ * α_p^2 * f_k && break
+                f_new ≤ criteria && break
             end
 
             α_tm = @. α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)

From feeb4386a1401178a0105ea762cae6e544d5f2c5 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Wed, 21 Jun 2023 23:49:30 +0200
Subject: [PATCH 124/375] itp method begin

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 1 +
 lib/SimpleNonlinearSolve/src/itp.jl                  | 0
 2 files changed, 1 insertion(+)
 create mode 100644 lib/SimpleNonlinearSolve/src/itp.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 8749aa72b..5074d449d 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -41,6 +41,7 @@ include("dfsane.jl")
 include("ad.jl")
 include("halley.jl")
 include("alefeld.jl")
+include("itp.jl")
 
 import PrecompileTools
 
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
new file mode 100644
index 000000000..e69de29bb

From c3132f0d9a015d186049ab5d364730859c796a00 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 23 Jun 2023 02:10:44 +0200
Subject: [PATCH 125/375] ipt alg

---
 lib/SimpleNonlinearSolve/src/itp.jl | 31 +++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index e69de29bb..49a565628 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -0,0 +1,31 @@
+"""
+```julia
+Itp(; k1 = Val{1}(), k2 = Val{2}(), n0 = Val{1}())
+```
+ITP (Interpolate Truncate & Project)
+
+
+"""
+
+struct Itp <: AbstractBracketingAlgorithm
+    k1::Real
+    k2::Real
+    n0::Int
+end
+
+function Itp(k1::Real = Val{1}(), k2::Real = Val{2}(), n0::Int = Val{1}())
+    if k1 < 0
+        ArgumentError("Hyper-parameter κ₁ should not be negative")
+    end
+    if !isa(n0, Int)
+        ArgumentError("Hyper-parameter n₀ should be an Integer")
+    end
+    Itp(k1, k2, n0)
+end
+
+function SciMLBase.__solve(prob::NonlinearProblem, alg::Itp,
+                            args..., abstol = nothing, reltol = nothing, 
+                            maxiters = 1000, kwargs...)
+
+                               
+end
\ No newline at end of file

From ab87f17a789f179366a0f276c8c0bb1a209b7de7 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 27 Jun 2023 01:26:46 +0200
Subject: [PATCH 126/375] minor fix

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 5074d449d..8177dd2e6 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -74,6 +74,6 @@ end
 
 # DiffEq styled algorithms
 export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
-    Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld
+    Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld, Itp
 
 end # module

From c3dab197778a1c5bf663475b123093fb5cec311c Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 27 Jun 2023 02:33:25 +0200
Subject: [PATCH 127/375] defined cache

---
 lib/SimpleNonlinearSolve/src/itp.jl | 31 ++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 49a565628..ff848484d 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -23,9 +23,38 @@ function Itp(k1::Real = Val{1}(), k2::Real = Val{2}(), n0::Int = Val{1}())
     Itp(k1, k2, n0)
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::Itp,
+function SciMLBase.__solve(prob::IntervalNonlinearProblem, alg::Itp,
                             args..., abstol = nothing, reltol = nothing, 
                             maxiters = 1000, kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    left, right = prob.tspan # a and b
+    fl, fr = f(left), f(right)
+    ϵ = abstol
+    if iszero(fl)
+        return SciMLBase.build_solution(prob, alg, left, fl;
+            retcode = ReturnCode.ExactSolutionLeft, left = left,
+            right = right)
+    end
+
+    if iszero(fr)
 
+    end
+    #defining variables/cache
+    k1 = alg.k1
+    k2 = alg.k2
+    n0 = alg.k3
+    n_h = ceil(log2((right - left) / (2 * ϵ)))
+    n_max = n_h + n0
+    mid = (left + right) / 2
+    x_f = (fr * left - fl * right) / (fr - fl)
+    r = zero(left)
+    δ = zero(left)
+    σ = sign(mid - x_f)
+    i = 0 #iteration
+    while i <= maxiters
+        mid = (left + right) / 2
+        r = ϵ * 2 ^ (n_max - i) - ((right - left) / 2)
+        δ = k1 * (right - left) ^ k2
+    end
                                
 end
\ No newline at end of file

From c1da36d6fa0d3fab0e1a2cf57e1fa92364c08d9c Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 27 Jun 2023 03:01:21 +0200
Subject: [PATCH 128/375] complete alg

---
 lib/SimpleNonlinearSolve/src/itp.jl | 51 ++++++++++++++++++++++++++---
 1 file changed, 47 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index ff848484d..2df125a74 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -47,14 +47,57 @@ function SciMLBase.__solve(prob::IntervalNonlinearProblem, alg::Itp,
     n_max = n_h + n0
     mid = (left + right) / 2
     x_f = (fr * left - fl * right) / (fr - fl)
-    r = zero(left)
-    δ = zero(left)
-    σ = sign(mid - x_f)
+    xt = left
+    xp = left
+    r = zero(left) #minmax radius
+    δ = zero(left) # truncation error
+    σ = 1.0
     i = 0 #iteration
     while i <= maxiters
-        mid = (left + right) / 2
+        #mid = (left + right) / 2
         r = ϵ * 2 ^ (n_max - i) - ((right - left) / 2)
         δ = k1 * (right - left) ^ k2
+
+        ## Interpolation step ##
+        x_f =  (fr * left - fl * right) / (fr - fl)
+
+        ## Truncation step ##
+        σ = sign(mid - x_f)
+        if δ <= abs(mid - x_f)
+            xt = x_f + (σ * δ)
+        else
+            xt = mid
+        end
+
+        ## Projection step ##
+        if abs(xt - mid) <= r
+            xp = xt
+        else
+            xp = mid - (σ * r)
+        end
+
+        ## Update ##
+        yp = f(xp)
+        if yp > 0
+            right = xp
+            fr = yp
+        elseif yp < 0
+            left = xp
+            fl = yp
+        else
+            left = xp
+            right = xp
+        end
+        i += 1
+        mid = (left + right) / 2
+
+        if (right - left < 2 * ϵ)
+            return SciMLBase.build_solution(prob, alg, mid, fl;
+            retcode = ReturnCode.Success, left = left,
+            right = right)
+        end
     end
+    return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
+        left = left, right = right)
                                
 end
\ No newline at end of file

From 6b9dee4d1c0cb4d087f2b5b909ab4c731d275b88 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 27 Jun 2023 03:09:56 +0200
Subject: [PATCH 129/375] hyperparams checks

---
 lib/SimpleNonlinearSolve/src/itp.jl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 2df125a74..853c3aa77 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -20,7 +20,13 @@ function Itp(k1::Real = Val{1}(), k2::Real = Val{2}(), n0::Int = Val{1}())
     if !isa(n0, Int)
         ArgumentError("Hyper-parameter n₀ should be an Integer")
     end
-    Itp(k1, k2, n0)
+    if n0 < 0
+        ArgumentError("Hyper-parameter n₀ should not be negative")
+    end
+    if k2 < 1 || k2 > (1.5 + sqrt(5) / 2)
+        ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where ϕ ≈ 1.618... is the golden ratio")
+    end
+    return Itp(k1, k2, n0)
 end
 
 function SciMLBase.__solve(prob::IntervalNonlinearProblem, alg::Itp,

From f6702fcce1c95bd0c4219419b06fa0692018522f Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 27 Jun 2023 03:16:58 +0200
Subject: [PATCH 130/375] abstol fix

---
 lib/SimpleNonlinearSolve/src/itp.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 853c3aa77..8eed6d92c 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -30,7 +30,7 @@ function Itp(k1::Real = Val{1}(), k2::Real = Val{2}(), n0::Int = Val{1}())
 end
 
 function SciMLBase.__solve(prob::IntervalNonlinearProblem, alg::Itp,
-                            args..., abstol = nothing, reltol = nothing, 
+                            args..., abstol = 1e-8, reltol = nothing, 
                             maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b

From 483a6d7da6f8c21c499f92ccbc992c43d6656821 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 27 Jun 2023 03:24:00 +0200
Subject: [PATCH 131/375] right retcode

---
 lib/SimpleNonlinearSolve/src/itp.jl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 8eed6d92c..014e321ca 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -40,10 +40,10 @@ function SciMLBase.__solve(prob::IntervalNonlinearProblem, alg::Itp,
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,
             right = right)
-    end
-
-    if iszero(fr)
-
+    elseif iszero(fr)
+        return SciMLBase.build_solution(prob, alg, right, fr;
+            retcode = ReturnCode.ExactSolutionRight, left = left,
+            right = right)
     end
     #defining variables/cache
     k1 = alg.k1

From 603106b7d70cfc9b1aa4855c4eb5305495505c4d Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Wed, 28 Jun 2023 03:25:27 +0200
Subject: [PATCH 132/375] update SimpleNonlinearSolve.jl

---
 .../src/SimpleNonlinearSolve.jl               |  2 +-
 lib/SimpleNonlinearSolve/src/itp.jl           | 31 +++++++++----------
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 8177dd2e6..ca787c245 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -66,7 +66,7 @@ PrecompileTools.@compile_workload begin
         prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
             T.((0.0, 2.0)),
             T(2))
-        for alg in (Bisection, Falsi, Ridder, Brent, Alefeld)
+        for alg in (Bisection, Falsi, Ridder, Brent, Alefeld, Itp)
             solve(prob_brack, alg(), abstol = T(1e-2))
         end
     end
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 014e321ca..085f80f53 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -11,26 +11,25 @@ struct Itp <: AbstractBracketingAlgorithm
     k1::Real
     k2::Real
     n0::Int
-end
-
-function Itp(k1::Real = Val{1}(), k2::Real = Val{2}(), n0::Int = Val{1}())
-    if k1 < 0
-        ArgumentError("Hyper-parameter κ₁ should not be negative")
-    end
-    if !isa(n0, Int)
-        ArgumentError("Hyper-parameter n₀ should be an Integer")
-    end
-    if n0 < 0
-        ArgumentError("Hyper-parameter n₀ should not be negative")
-    end
-    if k2 < 1 || k2 > (1.5 + sqrt(5) / 2)
-        ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where ϕ ≈ 1.618... is the golden ratio")
+    function Itp(;k1::Real = 0.1, k2::Real = 2.0, n0::Int = 1)
+        if k1 < 0
+            ArgumentError("Hyper-parameter κ₁ should not be negative")
+        end
+        if !isa(n0, Int)
+            ArgumentError("Hyper-parameter n₀ should be an Integer")
+        end
+        if n0 < 0
+            ArgumentError("Hyper-parameter n₀ should not be negative")
+        end
+        if k2 < 1 || k2 > (1.5 + sqrt(5) / 2)
+            ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where ϕ ≈ 1.618... is the golden ratio")
+        end
+        return new(k1, k2, n0)
     end
-    return Itp(k1, k2, n0)
 end
 
 function SciMLBase.__solve(prob::IntervalNonlinearProblem, alg::Itp,
-                            args..., abstol = 1e-8, reltol = nothing, 
+                            args...; abstol = nothing,
                             maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b

From 8882c457c4f66ad6cf8ddc69d3032cea1809a64a Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 28 Jun 2023 09:00:35 -0400
Subject: [PATCH 133/375] Update src/itp.jl

---
 lib/SimpleNonlinearSolve/src/itp.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 085f80f53..bd5f888dd 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -28,7 +28,7 @@ struct Itp <: AbstractBracketingAlgorithm
     end
 end
 
-function SciMLBase.__solve(prob::IntervalNonlinearProblem, alg::Itp,
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
                             args...; abstol = nothing,
                             maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)

From a4069d2e3180b773c205a3faaff624df4d4b15ef Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 27 Jun 2023 11:21:41 -0400
Subject: [PATCH 134/375] Use PackageExtensionCompat

---
 lib/SimpleNonlinearSolve/Project.toml                | 10 +++++-----
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 11 ++---------
 2 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index cf5a675af..d2551032d 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -9,10 +9,10 @@ DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+PackageExtensionCompat = "65ce6f38-6b18-4e1d-a461-8949797d7930"
+PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
-Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
-PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [weakdeps]
@@ -23,14 +23,14 @@ SimpleBatchedNonlinearSolveExt = "NNlib"
 
 [compat]
 ArrayInterface = "6, 7"
-DiffEqBase = "6.123.0"
+DiffEqBase = "6.126"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 NNlib = "0.8"
+PackageExtensionCompat = "1"
+PrecompileTools = "1"
 Reexport = "0.2, 1"
-Requires = "1"
 SciMLBase = "1.73"
-PrecompileTools = "1"
 StaticArraysCore = "1.4"
 julia = "1.6"
 
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 8749aa72b..97ce20379 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -10,16 +10,9 @@ using DiffEqBase
 
 @reexport using SciMLBase
 
-if !isdefined(Base, :get_extension)
-    using Requires
-end
-
+using PackageExtensionCompat
 function __init__()
-    @static if !isdefined(Base, :get_extension)
-        @require NNlib="872c559c-99b0-510c-b3b7-b6c96a88d5cd" begin
-            include("../ext/SimpleBatchedNonlinearSolveExt.jl")
-        end
-    end
+    @require_extensions
 end
 
 abstract type AbstractSimpleNonlinearSolveAlgorithm <: SciMLBase.AbstractNonlinearAlgorithm end

From 4e2de7a1877b05dda15fa16b4d03681e79fb1c3e Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 27 Jun 2023 11:51:36 -0400
Subject: [PATCH 135/375] Rework BatchedBroyden to be more efficient

---
 lib/SimpleNonlinearSolve/Project.toml         |   4 +-
 .../ext/SimpleBatchedNonlinearSolveExt.jl     | 120 ++++++++----------
 .../src/SimpleNonlinearSolve.jl               |   9 ++
 .../src/batched/broyden.jl                    |  20 +++
 .../src/batched/dfsane.jl                     |   0
 .../src/batched/lbroyden.jl                   |   0
 .../src/batched/raphson.jl                    |   0
 lib/SimpleNonlinearSolve/src/batched/utils.jl |  78 ++++++++++++
 lib/SimpleNonlinearSolve/src/broyden.jl       |  16 +--
 lib/SimpleNonlinearSolve/test/basictests.jl   |  12 +-
 10 files changed, 176 insertions(+), 83 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/batched/broyden.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/batched/dfsane.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/batched/raphson.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/batched/utils.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index d2551032d..b1b3a422c 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.16"
+version = "0.1.17"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
@@ -26,7 +26,7 @@ ArrayInterface = "6, 7"
 DiffEqBase = "6.126"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
-NNlib = "0.8"
+NNlib = "0.8, 0.9"
 PackageExtensionCompat = "1"
 PrecompileTools = "1"
 Reexport = "0.2, 1"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
index dd7628888..9e1ba59ad 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
@@ -1,90 +1,76 @@
 module SimpleBatchedNonlinearSolveExt
 
-using ArrayInterface, DiffEqBase, LinearAlgebra, SimpleNonlinearSolve, SciMLBase
+using ArrayInterface, DiffEqBase, LinearAlgebra, SimpleNonlinearSolve, SciMLBase, NNlib
+import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage, _init_𝓙, _result_from_storage, _get_tolerance, @maybeinplace
 
-isdefined(Base, :get_extension) ? (using NNlib) : (using ..NNlib)
+@views function SciMLBase.__solve(prob::NonlinearProblem,
+    alg::BatchedBroyden;
+    abstol=nothing,
+    reltol=nothing,
+    maxiters=1000,
+    kwargs...)
+    iip = isinplace(prob)
+    u0 = prob.u0
 
-_batch_transpose(x) = reshape(x, 1, size(x)...)
+    u, f, reconstruct = _construct_batched_problem_structure(prob)
+    L, N = size(u)
 
-_batched_mul(x, y) = x * y
-
-function _batched_mul(x::AbstractArray{T, 3}, y::AbstractMatrix) where {T}
-    return dropdims(batched_mul(x, reshape(y, size(y, 1), 1, size(y, 2))); dims = 2)
-end
-
-function _batched_mul(x::AbstractMatrix, y::AbstractArray{T, 3}) where {T}
-    return batched_mul(reshape(x, size(x, 1), 1, size(x, 2)), y)
-end
-
-function _batched_mul(x::AbstractArray{T1, 3}, y::AbstractArray{T2, 3}) where {T1, T2}
-    return batched_mul(x, y)
-end
-
-function _init_J_batched(x::AbstractMatrix{T}) where {T}
-    J = ArrayInterface.zeromatrix(x[:, 1])
-    if ismutable(x)
-        J[diagind(J)] .= one(eltype(x))
-    else
-        J += I
-    end
-    return repeat(J, 1, 1, size(x, 2))
-end
-
-function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{true}, args...;
-    abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
-    f = Base.Fix2(prob.f, prob.p)
-    x = float(prob.u0)
 
-    if ndims(x) != 2
-        error("`batch` mode works only if `ndims(prob.u0) == 2`")
-    end
+    storage = _get_storage(mode, u)
 
-    fₙ = f(x)
-    T = eltype(x)
-    J⁻¹ = _init_J_batched(x)
+    xₙ, xₙ₋₁, δx, δf = ntuple(_ -> copy(u), 4)
+    T = eltype(u)
 
-    if SciMLBase.isinplace(prob)
-        error("Broyden currently only supports out-of-place nonlinear problems")
-    end
+    atol = _get_tolerance(abstol, tc.abstol, T)
+    rtol = _get_tolerance(reltol, tc.reltol, T)
+    termination_condition = tc(storage)
 
-    atol = abstol !== nothing ? abstol :
-           (tc.abstol !== nothing ? tc.abstol :
-            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
-    rtol = reltol !== nothing ? reltol :
-           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
+    𝓙⁻¹ = _init_𝓙(xₙ)  # L × L × N
+    𝓙⁻¹f, xᵀ𝓙⁻¹δf, xᵀ𝓙⁻¹ = similar(𝓙⁻¹, L, N), similar(𝓙⁻¹, 1, N), similar(𝓙⁻¹, 1, L, N)
 
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        error("Broyden currently doesn't support SAFE_BEST termination modes")
-    end
+    @maybeinplace iip fₙ₋₁=f(xₙ) u
+    iip && (fₙ = copy(fₙ₋₁))
+    for n in 1:maxiters
+        batched_mul!(reshape(𝓙⁻¹f, L, 1, N), 𝓙⁻¹, reshape(fₙ₋₁, L, 1, N))
+        xₙ .= xₙ₋₁ .- 𝓙⁻¹f
 
-    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
-              nothing
-    termination_condition = tc(storage)
+        @maybeinplace iip fₙ=f(xₙ)
+        δx .= xₙ .- xₙ₋₁
+        δf .= fₙ .- fₙ₋₁
+
+        batched_mul!(reshape(𝓙⁻¹f, L, 1, N), 𝓙⁻¹, reshape(δf, L, 1, N))
+        δxᵀ = reshape(δx, 1, L, N)
 
-    xₙ = x
-    xₙ₋₁ = x
-    fₙ₋₁ = fₙ
-    for i in 1:maxiters
-        xₙ = xₙ₋₁ .- _batched_mul(J⁻¹, fₙ₋₁)
-        fₙ = f(xₙ)
-        Δxₙ = xₙ .- xₙ₋₁
-        Δfₙ = fₙ .- fₙ₋₁
-        J⁻¹Δfₙ = _batched_mul(J⁻¹, Δfₙ)
-        J⁻¹ += _batched_mul(((Δxₙ .- J⁻¹Δfₙ) ./
-                             (_batched_mul(_batch_transpose(Δxₙ), J⁻¹Δfₙ) .+ T(1e-5))),
-            _batched_mul(_batch_transpose(Δxₙ), J⁻¹))
+        batched_mul!(reshape(xᵀ𝓙⁻¹δf, 1, 1, N), δxᵀ, reshape(𝓙⁻¹f, L, 1, N))
+        batched_mul!(xᵀ𝓙⁻¹, δxᵀ, 𝓙⁻¹)
+        δx .= (δx .- 𝓙⁻¹f) ./ (xᵀ𝓙⁻¹δf .+ T(1e-5))
+        batched_mul!(𝓙⁻¹, reshape(δx, L, 1, N), xᵀ𝓙⁻¹, one(T), one(T))
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
+            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode)
+            return DiffEqBase.build_solution(prob,
+                alg,
+                reconstruct(xₙ),
+                reconstruct(fₙ);
+                retcode)
         end
 
-        xₙ₋₁ = xₙ
-        fₙ₋₁ = fₙ
+        xₙ₋₁ .= xₙ
+        fₙ₋₁ .= fₙ
+    end
+
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        xₙ = storage.u
+        @maybeinplace iip fₙ=f(xₙ)
     end
 
-    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+    return DiffEqBase.build_solution(prob,
+        alg,
+        reconstruct(xₙ),
+        reconstruct(fₙ);
+        retcode=ReturnCode.MaxIters)
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 97ce20379..ab2033622 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -19,6 +19,7 @@ abstract type AbstractSimpleNonlinearSolveAlgorithm <: SciMLBase.AbstractNonline
 abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 abstract type AbstractNewtonAlgorithm{CS, AD, FDT} <: AbstractSimpleNonlinearSolveAlgorithm end
 abstract type AbstractImmutableNonlinearSolver <: AbstractSimpleNonlinearSolveAlgorithm end
+abstract type AbstractBatchedNonlinearSolveAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 
 include("utils.jl")
 include("bisection.jl")
@@ -35,6 +36,13 @@ include("ad.jl")
 include("halley.jl")
 include("alefeld.jl")
 
+# Batched Solver Support
+include("batched/utils.jl")
+include("batched/raphson.jl")
+include("batched/dfsane.jl")
+include("batched/broyden.jl")
+include("batched/lbroyden.jl")
+
 import PrecompileTools
 
 PrecompileTools.@compile_workload begin
@@ -67,5 +75,6 @@ end
 # DiffEq styled algorithms
 export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
     Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld
+export BatchedBroyden
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/batched/broyden.jl b/lib/SimpleNonlinearSolve/src/batched/broyden.jl
new file mode 100644
index 000000000..a301c4e14
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/batched/broyden.jl
@@ -0,0 +1,20 @@
+"""
+    BatchedBroyden(;
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing))
+
+A low-overhead batched implementation of Broyden capable of solving multiple nonlinear
+problems simultaneously.
+
+!!! note
+
+    To use this version, remember to load `NNlib`, i.e., `using NNlib` or
+    `import NNlib` must be present in your code.
+"""
+struct BatchedBroyden{TC <: NLSolveTerminationCondition} <:
+    AbstractBatchedNonlinearSolveAlgorithm
+    termination_condition::TC
+end
+
+# Implementation of solve using Package Extensions
diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl b/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/SimpleNonlinearSolve/src/batched/raphson.jl b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/SimpleNonlinearSolve/src/batched/utils.jl b/lib/SimpleNonlinearSolve/src/batched/utils.jl
new file mode 100644
index 000000000..9ed91345f
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/batched/utils.jl
@@ -0,0 +1,78 @@
+function _get_tolerance(η, tc_η, ::Type{T}) where {T}
+    fallback_η = real(oneunit(T)) * (eps(real(one(T))))^(4 // 5)
+    return ifelse(η !== nothing, η, ifelse(tc_η !== nothing, tc_η, fallback_η))
+end
+
+function _construct_batched_problem_structure(prob)
+    return _construct_batched_problem_structure(prob.u0,
+        prob.f,
+        prob.p,
+        Val(SciMLBase.isinplace(prob)))
+end
+
+function _construct_batched_problem_structure(u0::AbstractArray{T, N},
+    f,
+    p,
+    ::Val{iip}) where {T, N, iip}
+    # Reconstruct `u`
+    reconstruct = N == 2 ? identity : Base.Fix2(reshape, size(u0))
+    # Standardize `u`
+    standardize = N == 2 ? identity :
+                  (N == 1 ? Base.Fix2(reshape, (:, 1)) :
+                   Base.Fix2(reshape, (:, size(u0, ndims(u0)))))
+    # Updated Function
+    f_modified = if iip
+        function f_modified_iip(du, u)
+            f(reconstruct(du), reconstruct(u), p)
+            return standardize(du)
+        end
+    else
+        f_modified_oop(u) = standardize(f(reconstruct(u), p))
+    end
+    return standardize(u0), f_modified, reconstruct
+end
+
+@views function _init_𝓙(x::AbstractMatrix)
+    𝓙 = ArrayInterface.zeromatrix(x[:, 1])
+    if ismutable(x)
+        𝓙[diagind(𝓙)] .= one(eltype(x))
+    else
+        𝓙 .+= I
+    end
+    return repeat(𝓙, 1, 1, size(x, 2))
+end
+
+_result_from_storage(::Nothing, xₙ, fₙ, f, mode) = ReturnCode.Success, xₙ, fₙ
+function _result_from_storage(storage::NLSolveSafeTerminationResult, xₙ, fₙ, f, mode)
+    if storage.return_code == DiffEqBase.NLSolveSafeTerminationReturnCode.Success
+        return ReturnCode.Success, xₙ, fₙ
+    else
+        if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+            return ReturnCode.Terminated, storage.u, f(storage.u)
+        else
+            return ReturnCode.Terminated, xₙ, fₙ
+        end
+    end
+end
+
+function _get_storage(mode, u)
+    return mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ?
+           NLSolveSafeTerminationResult(mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES ? u :
+                                        nothing) : nothing
+end
+
+macro maybeinplace(iip::Symbol, expr::Expr, u0::Union{Symbol, Nothing}=nothing)
+    @assert expr.head == :(=)
+    x1, x2 = expr.args
+    @assert x2.head == :call
+    f, x = x2.args
+    define_expr = u0 === nothing ? :() : :($(x1) = similar($(u0)))
+    return quote
+        if $(esc(iip))
+            $(esc(define_expr))
+            $(esc(f))($(esc(x1)), $(esc(x)))
+        else
+            $(esc(expr))
+        end
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 8ce0d66b5..56fa2efea 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -11,19 +11,19 @@ and static array problems.
     To use the `batched` version, remember to load `NNlib`, i.e., `using NNlib` or
     `import NNlib` must be present in your code.
 """
-struct Broyden{batched, TC <: NLSolveTerminationCondition} <:
+struct Broyden{TC <: NLSolveTerminationCondition} <:
        AbstractSimpleNonlinearSolveAlgorithm
     termination_condition::TC
+end
 
-    function Broyden(; batched = false,
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing))
-        return new{batched, typeof(termination_condition)}(termination_condition)
-    end
+function Broyden(; batched = false,
+    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+        abstol = nothing,
+        reltol = nothing))
+    return (batched ? BatchedBroyden : Broyden)(termination_condition)
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden{false}, args...;
+function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
     abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index aea173157..dd54527a5 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -5,12 +5,12 @@ using DiffEqBase
 using LinearAlgebra
 using Test
 
-const BATCHED_BROYDEN_SOLVERS = Broyden[]
-const BROYDEN_SOLVERS = Broyden[]
-const BATCHED_LBROYDEN_SOLVERS = LBroyden[]
-const LBROYDEN_SOLVERS = LBroyden[]
-const BATCHED_DFSANE_SOLVERS = SimpleDFSane[]
-const DFSANE_SOLVERS = SimpleDFSane[]
+const BATCHED_BROYDEN_SOLVERS = []
+const BROYDEN_SOLVERS = []
+const BATCHED_LBROYDEN_SOLVERS = []
+const LBROYDEN_SOLVERS = []
+const BATCHED_DFSANE_SOLVERS = []
+const DFSANE_SOLVERS = []
 
 for mode in instances(NLSolveTerminationMode.T)
     if mode ∈

From dcd54a77cde48207983ced207954319febf9dbca Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 27 Jun 2023 15:29:45 -0400
Subject: [PATCH 136/375] Add Batching to Newton Raphson

---
 lib/SimpleNonlinearSolve/Project.toml         |  9 +-
 .../SimpleNonlinearSolveADLinearSolveExt.jl   | 89 +++++++++++++++++++
 ...Ext.jl => SimpleNonlinearSolveNNlibExt.jl} | 12 ++-
 .../src/SimpleNonlinearSolve.jl               |  6 +-
 .../src/batched/raphson.jl                    | 24 +++++
 lib/SimpleNonlinearSolve/src/batched/utils.jl | 39 ++++----
 lib/SimpleNonlinearSolve/src/broyden.jl       | 11 ++-
 lib/SimpleNonlinearSolve/src/raphson.jl       | 42 +++++++--
 8 files changed, 195 insertions(+), 37 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
 rename lib/SimpleNonlinearSolve/ext/{SimpleBatchedNonlinearSolveExt.jl => SimpleNonlinearSolveNNlibExt.jl} (90%)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index b1b3a422c..901708505 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -16,16 +16,21 @@ SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [weakdeps]
+AbstractDifferentiation = "c29ec348-61ec-40c8-8164-b8c60e9d9f3d"
+LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 
 [extensions]
-SimpleBatchedNonlinearSolveExt = "NNlib"
+SimpleNonlinearSolveNNlibExt = "NNlib"
+SimpleNonlinearSolveADLinearSolveExt = ["AbstractDifferentiation", "LinearSolve"]
 
 [compat]
+AbstractDifferentiation = "0.5"
 ArrayInterface = "6, 7"
 DiffEqBase = "6.126"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
+LinearSolve = "2"
 NNlib = "0.8, 0.9"
 PackageExtensionCompat = "1"
 PrecompileTools = "1"
@@ -35,7 +40,9 @@ StaticArraysCore = "1.4"
 julia = "1.6"
 
 [extras]
+AbstractDifferentiation = "c29ec348-61ec-40c8-8164-b8c60e9d9f3d"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
new file mode 100644
index 000000000..96ba9168a
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
@@ -0,0 +1,89 @@
+module SimpleNonlinearSolveADLinearSolveExt
+
+using AbstractDifferentiation, ArrayInterface, DiffEqBase, LinearAlgebra, LinearSolve,
+    SimpleNonlinearSolve, SciMLBase
+import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage, _result_from_storage, _get_tolerance, @maybeinplace
+
+const AD = AbstractDifferentiation
+
+function __init__()
+    SimpleNonlinearSolve.ADLinearSolveExtLoaded[] = true
+    return
+end
+
+function SimpleNonlinearSolve.SimpleBatchedNewtonRaphson(; chunk_size = Val{0}(),
+    autodiff = Val{true}(),
+    diff_type = Val{:forward},
+    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+        abstol = nothing,
+        reltol = nothing))
+    # TODO: Use `diff_type`. FiniteDiff.jl is currently not available in AD.jl
+    chunksize = SciMLBase._unwrap_val(chunk_size) == 0 ? nothing : chunk_size
+    ad = SciMLBase._unwrap_val(autodiff) ?
+        AD.ForwardDiffBackend(; chunksize) :
+        AD.FiniteDifferencesBackend()
+    return SimpleBatchedNewtonRaphson{typeof(ad), Nothing, typeof(termination_condition)}(
+        ad,
+        nothing,
+        termination_condition)
+end
+
+function SciMLBase.__solve(prob::NonlinearProblem,
+    alg::SimpleBatchedNewtonRaphson;
+    abstol=nothing,
+    reltol=nothing,
+    maxiters=1000,
+    kwargs...)
+    iip = isinplace(prob)
+    @assert !iip "SimpleBatchedNewtonRaphson currently only supports out-of-place nonlinear problems."
+    u, f, reconstruct = _construct_batched_problem_structure(prob)
+
+    tc = alg.termination_condition
+    mode = DiffEqBase.get_termination_mode(tc)
+
+    storage = _get_storage(mode, u)
+
+    xₙ, xₙ₋₁, δx = copy(u), copy(u), copy(u)
+    T = eltype(u)
+
+    atol = _get_tolerance(abstol, tc.abstol, T)
+    rtol = _get_tolerance(reltol, tc.reltol, T)
+    termination_condition = tc(storage)
+
+    for i in 1:maxiters
+        fₙ, (𝓙,) = AD.value_and_jacobian(alg.autodiff, f, xₙ)
+
+        iszero(fₙ) && return DiffEqBase.build_solution(prob,
+            alg,
+            reconstruct(xₙ),
+            reconstruct(fₙ);
+            retcode=ReturnCode.Success)
+
+        solve(LinearProblem(𝓙, vec(fₙ); u0=vec(δx)), alg.linsolve; kwargs...)
+        xₙ .-= δx
+
+        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
+            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
+            return DiffEqBase.build_solution(prob,
+                alg,
+                reconstruct(xₙ),
+                reconstruct(fₙ);
+                retcode)
+        end
+
+        xₙ₋₁ .= xₙ
+    end
+
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        xₙ = storage.u
+        fₙ = f(xₙ)
+    end
+
+    return DiffEqBase.build_solution(prob,
+        alg,
+        reconstruct(xₙ),
+        reconstruct(fₙ);
+        retcode=ReturnCode.MaxIters)
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
similarity index 90%
rename from lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
rename to lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
index 9e1ba59ad..e62e2bda7 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleBatchedNonlinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
@@ -1,8 +1,13 @@
-module SimpleBatchedNonlinearSolveExt
+module SimpleNonlinearSolveNNlibExt
 
-using ArrayInterface, DiffEqBase, LinearAlgebra, SimpleNonlinearSolve, SciMLBase, NNlib
+using ArrayInterface, DiffEqBase, LinearAlgebra, NNlib, SimpleNonlinearSolve, SciMLBase
 import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage, _init_𝓙, _result_from_storage, _get_tolerance, @maybeinplace
 
+function __init__()
+    SimpleNonlinearSolve.NNlibExtLoaded[] = true
+    return
+end
+
 @views function SciMLBase.__solve(prob::NonlinearProblem,
     alg::BatchedBroyden;
     abstol=nothing,
@@ -10,7 +15,6 @@ import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage,
     maxiters=1000,
     kwargs...)
     iip = isinplace(prob)
-    u0 = prob.u0
 
     u, f, reconstruct = _construct_batched_problem_structure(prob)
     L, N = size(u)
@@ -49,7 +53,7 @@ import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage,
         batched_mul!(𝓙⁻¹, reshape(δx, L, 1, N), xᵀ𝓙⁻¹, one(T), one(T))
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
-            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode)
+            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
             return DiffEqBase.build_solution(prob,
                 alg,
                 reconstruct(xₙ),
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index ab2033622..49aef9fc8 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -15,6 +15,9 @@ function __init__()
     @require_extensions
 end
 
+const ADLinearSolveExtLoaded = Ref{Bool}(false)
+const NNlibExtLoaded = Ref{Bool}(false)
+
 abstract type AbstractSimpleNonlinearSolveAlgorithm <: SciMLBase.AbstractNonlinearAlgorithm end
 abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 abstract type AbstractNewtonAlgorithm{CS, AD, FDT} <: AbstractSimpleNonlinearSolveAlgorithm end
@@ -75,6 +78,7 @@ end
 # DiffEq styled algorithms
 export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
     Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld
-export BatchedBroyden
+export BatchedBroyden, SimpleBatchedNewtonRaphson, SimpleBatchedDFSane,
+    BatchedLBroyden
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/batched/raphson.jl b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
index e69de29bb..0a28e338f 100644
--- a/lib/SimpleNonlinearSolve/src/batched/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
@@ -0,0 +1,24 @@
+"""
+    SimpleBatchedNewtonRaphson(; chunk_size = Val{0}(),
+        autodiff = Val{true}(),
+        diff_type = Val{:forward},
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing))
+
+A low-overhead batched implementation of Newton-Raphson capable of solving multiple
+nonlinear problems simultaneously.
+
+!!! note
+
+    To use the `batched` version, remember to load `AbstractDifferentiation` and
+    `LinearSolve`.
+"""
+struct SimpleBatchedNewtonRaphson{AD, LS, TC <: NLSolveTerminationCondition} <:
+    AbstractBatchedNonlinearSolveAlgorithm
+    autodiff::AD
+    linsolve::LS
+    termination_condition::TC
+end
+
+# Implementation of solve using Package Extensions
diff --git a/lib/SimpleNonlinearSolve/src/batched/utils.jl b/lib/SimpleNonlinearSolve/src/batched/utils.jl
index 9ed91345f..33441e985 100644
--- a/lib/SimpleNonlinearSolve/src/batched/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/utils.jl
@@ -1,3 +1,19 @@
+macro maybeinplace(iip::Symbol, expr::Expr, u0::Union{Symbol, Nothing}=nothing)
+    @assert expr.head == :(=)
+    x1, x2 = expr.args
+    @assert x2.head == :call
+    f, x = x2.args
+    define_expr = u0 === nothing ? :() : :($(x1) = similar($(u0)))
+    return quote
+        if $(esc(iip))
+            $(esc(define_expr))
+            $(esc(f))($(esc(x1)), $(esc(x)))
+        else
+            $(esc(expr))
+        end
+    end
+end
+
 function _get_tolerance(η, tc_η, ::Type{T}) where {T}
     fallback_η = real(oneunit(T)) * (eps(real(one(T))))^(4 // 5)
     return ifelse(η !== nothing, η, ifelse(tc_η !== nothing, tc_η, fallback_η))
@@ -42,13 +58,14 @@ end
     return repeat(𝓙, 1, 1, size(x, 2))
 end
 
-_result_from_storage(::Nothing, xₙ, fₙ, f, mode) = ReturnCode.Success, xₙ, fₙ
-function _result_from_storage(storage::NLSolveSafeTerminationResult, xₙ, fₙ, f, mode)
+_result_from_storage(::Nothing, xₙ, fₙ, args...) = ReturnCode.Success, xₙ, fₙ
+function _result_from_storage(storage::NLSolveSafeTerminationResult, xₙ, fₙ, f, mode, iip)
     if storage.return_code == DiffEqBase.NLSolveSafeTerminationReturnCode.Success
         return ReturnCode.Success, xₙ, fₙ
     else
         if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-            return ReturnCode.Terminated, storage.u, f(storage.u)
+            @maybeinplace iip fₙ = f(xₙ)
+            return ReturnCode.Terminated, storage.u, fₙ
         else
             return ReturnCode.Terminated, xₙ, fₙ
         end
@@ -60,19 +77,3 @@ function _get_storage(mode, u)
            NLSolveSafeTerminationResult(mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES ? u :
                                         nothing) : nothing
 end
-
-macro maybeinplace(iip::Symbol, expr::Expr, u0::Union{Symbol, Nothing}=nothing)
-    @assert expr.head == :(=)
-    x1, x2 = expr.args
-    @assert x2.head == :call
-    f, x = x2.args
-    define_expr = u0 === nothing ? :() : :($(x1) = similar($(u0)))
-    return quote
-        if $(esc(iip))
-            $(esc(define_expr))
-            $(esc(f))($(esc(x1)), $(esc(x)))
-        else
-            $(esc(expr))
-        end
-    end
-end
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 56fa2efea..74566e8c3 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -1,7 +1,8 @@
 """
     Broyden(; batched = false,
-            termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-                                                                abstol = nothing, reltol = nothing))
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing))
 
 A low-overhead implementation of Broyden. This method is non-allocating on scalar
 and static array problems.
@@ -20,7 +21,11 @@ function Broyden(; batched = false,
     termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
         abstol = nothing,
         reltol = nothing))
-    return (batched ? BatchedBroyden : Broyden)(termination_condition)
+    if batched
+        @assert NNlibExtLoaded[] "Please install and load `NNlib.jl` to use batched Broyden."
+        return BatchedBroyden(termination_condition)
+    end
+    return Broyden(termination_condition)
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 386c35053..59eb1ed6d 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -1,8 +1,8 @@
 """
-```julia
-SimpleNewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
-                                 diff_type = Val{:forward})
-```
+    SimpleNewtonRaphson(; batched = false,
+        chunk_size = Val{0}(),
+        autodiff = Val{true}(),
+        diff_type = Val{:forward})
 
 A low-overhead implementation of Newton-Raphson. This method is non-allocating on scalar
 and static array problems.
@@ -27,13 +27,37 @@ and static array problems.
 - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
   `Val{:forward}` for forward finite differences. For more details on the choices, see the
   [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+
+!!! note
+
+    To use the `batched` version, remember to load `AbstractDifferentiation` and
+    `LinearSolve`.
 """
-struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
-    function SimpleNewtonRaphson(; chunk_size = Val{0}(), autodiff = Val{true}(),
-        diff_type = Val{:forward})
-        new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
-            SciMLBase._unwrap_val(diff_type)}()
+struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT} end
+
+function SimpleNewtonRaphson(; batched=false,
+    chunk_size = Val{0}(),
+    autodiff = Val{true}(),
+    diff_type = Val{:forward},
+    termination_condition = missing)
+    if !ismissing(termination_condition) && !batched
+        throw(ArgumentError("`termination_condition` is currently only supported for batched problems"))
+    end
+    if batched
+        @assert ADLinearSolveExtLoaded[] "Please install and load `LinearSolve.jl` and `AbstractDifferentiation.jl` to use batched Newton-Raphson."
+        termination_condition = ismissing(termination_condition) ?
+            NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+                abstol = nothing,
+                reltol = nothing) :
+            termination_condition
+        return SimpleBatchedNewtonRaphson(; chunk_size,
+            autodiff,
+            diff_type,
+            termination_condition)
     end
+    return SimpleNewtonRaphson{SciMLBase._unwrap_val(chunk_size),
+        SciMLBase._unwrap_val(autodiff),
+        SciMLBase._unwrap_val(diff_type)}()
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,

From 51005e053741fa5c82601415ee70d6074aa3404c Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 27 Jun 2023 16:59:54 -0400
Subject: [PATCH 137/375] Update SimpleDFSane Batching

---
 lib/SimpleNonlinearSolve/Project.toml         |   8 -
 .../src/batched/dfsane.jl                     | 140 ++++++++++++++++++
 lib/SimpleNonlinearSolve/src/batched/utils.jl |   4 +-
 lib/SimpleNonlinearSolve/src/broyden.jl       |   7 +-
 lib/SimpleNonlinearSolve/src/dfsane.jl        | 122 ++++++---------
 lib/SimpleNonlinearSolve/test/Project.toml    |  12 ++
 lib/SimpleNonlinearSolve/test/basictests.jl   |  13 +-
 7 files changed, 201 insertions(+), 105 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/test/Project.toml

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 901708505..f1f472d08 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -41,13 +41,5 @@ julia = "1.6"
 
 [extras]
 AbstractDifferentiation = "c29ec348-61ec-40c8-8164-b8c60e9d9f3d"
-BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["BenchmarkTools", "SafeTestsets", "Pkg", "Test", "StaticArrays", "NNlib"]
diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
index e69de29bb..fe7cbcddf 100644
--- a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
@@ -0,0 +1,140 @@
+@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition} <:
+              AbstractBatchedNonlinearSolveAlgorithm
+    σₘᵢₙ::T = 1.0f-10
+    σₘₐₓ::T = 1.0f+10
+    σ₁::T = 1.0f0
+    M::Int = 10
+    γ::T = 1.0f-4
+    τₘᵢₙ::T = 0.1f0
+    τₘₐₓ::T = 0.5f0
+    nₑₓₚ::Int = 2
+    ηₛ::F = (f₍ₙₒᵣₘ₎₁, n, xₙ, fₙ) -> f₍ₙₒᵣₘ₎₁ ./ n .^ 2
+    termination_condition::TC = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+        abstol=nothing,
+        reltol=nothing)
+    max_inner_iterations::Int = 1000
+end
+
+function SciMLBase.__solve(prob::NonlinearProblem,
+    alg::SimpleBatchedDFSane,
+    args...;
+    abstol=nothing,
+    reltol=nothing,
+    maxiters=100,
+    kwargs...)
+    iip = isinplace(prob)
+
+    u, f, reconstruct = _construct_batched_problem_structure(prob)
+    L, N = size(u)
+    T = eltype(u)
+
+    tc = alg.termination_condition
+    mode = DiffEqBase.get_termination_mode(tc)
+
+    storage = _get_storage(mode, u)
+
+    atol = _get_tolerance(abstol, tc.abstol, T)
+    rtol = _get_tolerance(reltol, tc.reltol, T)
+    termination_condition = tc(storage)
+
+    σₘᵢₙ, σₘₐₓ, γ, τₘᵢₙ, τₘₐₓ = T(alg.σₘᵢₙ), T(alg.σₘₐₓ), T(alg.γ), T(alg.τₘᵢₙ), T(alg.τₘₐₓ)
+    α₁ = one(T)
+    α₊, α₋ = similar(u, 1, N), similar(u, 1, N)
+    σₙ = fill(T(alg.σ₁), 1, N)
+    𝒹 = similar(σₙ, L, N)
+    (; M, nₑₓₚ) = alg
+
+    xₙ, xₙ₋₁, f₍ₙₒᵣₘ₎ₙ₋₁, f₍ₙₒᵣₘ₎ₙ = copy(u), copy(u), similar(u, 1, N), similar(u, 1, N)
+
+    function ff!(fₓ, fₙₒᵣₘ, x)
+        f(fₓ, x)
+        sum!(abs2, fₙₒᵣₘ, fₓ)
+        fₙₒᵣₘ .^= (nₑₓₚ / 2)
+        return fₓ
+    end
+
+    function ff!(fₙₒᵣₘ, x)
+        fₓ = f(x)
+        sum!(abs2, fₙₒᵣₘ, fₓ)
+        fₙₒᵣₘ .^= (nₑₓₚ / 2)
+        return fₓ
+    end
+
+    @maybeinplace iip fₙ₋₁ = ff!(f₍ₙₒᵣₘ₎ₙ₋₁, xₙ) xₙ
+    iip && (fₙ = similar(fₙ₋₁))
+    ℋ = repeat(f₍ₙₒᵣₘ₎ₙ₋₁, M, 1)
+    f̄ = similar(ℋ, 1, N)
+    ηₛ = (n, xₙ, fₙ) -> alg.ηₛ(f₍ₙₒᵣₘ₎ₙ₋₁, n, xₙ, fₙ)
+
+    for n in 1:maxiters
+        # Spectral parameter range check
+        @. σₙ = sign(σₙ) * clamp(abs(σₙ), σₘᵢₙ, σₘₐₓ)
+
+        # Line search direction
+        @. 𝒹 = -σₙ * fₙ₋₁
+
+        η = ηₛ(n, xₙ₋₁, fₙ₋₁)
+        maximum!(f̄, ℋ)
+        fill!(α₊, α₁)
+        fill!(α₋, α₁)
+        @. xₙ = xₙ₋₁ + α₊ * 𝒹
+
+        @maybeinplace iip fₙ = ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
+
+        for _ in 1:(alg.max_inner_iterations)
+            𝒸 = @. f̄ + η - γ * α₊^2 * f₍ₙₒᵣₘ₎ₙ₋₁
+
+            (sum(f₍ₙₒᵣₘ₎ₙ .≤ 𝒸) ≥ N ÷ 2) && break
+
+            @. α₊ = clamp(α₊^2 * f₍ₙₒᵣₘ₎ₙ₋₁ / (f₍ₙₒᵣₘ₎ₙ + (T(2) * α₊ - T(1)) * f₍ₙₒᵣₘ₎ₙ₋₁),
+                τₘᵢₙ * α₊,
+                τₘₐₓ * α₊)
+            @. xₙ = xₙ₋₁ - α₋ * 𝒹
+            @maybeinplace iip fₙ = ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
+
+            (sum(f₍ₙₒᵣₘ₎ₙ .≤ 𝒸) ≥ N ÷ 2) && break
+
+            @. α₋ = clamp(α₋^2 * f₍ₙₒᵣₘ₎ₙ₋₁ / (f₍ₙₒᵣₘ₎ₙ + (T(2) * α₋ - T(1)) * f₍ₙₒᵣₘ₎ₙ₋₁),
+                τₘᵢₙ * α₋,
+                τₘₐₓ * α₋)
+            @. xₙ = xₙ₋₁ + α₊ * 𝒹
+            @maybeinplace iip fₙ = ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
+        end
+
+        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
+            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
+            return DiffEqBase.build_solution(prob,
+                alg,
+                reconstruct(xₙ),
+                reconstruct(fₙ);
+                retcode)
+        end
+
+        # Update spectral parameter
+        @. xₙ₋₁ = xₙ - xₙ₋₁
+        @. fₙ₋₁ = fₙ - fₙ₋₁
+
+        sum!(abs2, α₊, xₙ₋₁)
+        sum!(α₋, xₙ₋₁ .* fₙ₋₁)
+        σₙ .= α₊ ./ (α₋ .+ T(1e-5))
+
+        # Take step
+        @. xₙ₋₁ = xₙ
+        @. fₙ₋₁ = fₙ
+        @. f₍ₙₒᵣₘ₎ₙ₋₁ = f₍ₙₒᵣₘ₎ₙ
+
+        # Update history
+        ℋ[n % M + 1, :] .= view(f₍ₙₒᵣₘ₎ₙ, 1, :)
+    end
+
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        xₙ = storage.u
+        @maybeinplace iip fₙ = f(xₙ)
+    end
+
+    return DiffEqBase.build_solution(prob,
+        alg,
+        reconstruct(xₙ),
+        reconstruct(fₙ);
+        retcode=ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/src/batched/utils.jl b/lib/SimpleNonlinearSolve/src/batched/utils.jl
index 33441e985..b5dbd597f 100644
--- a/lib/SimpleNonlinearSolve/src/batched/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/utils.jl
@@ -2,12 +2,12 @@ macro maybeinplace(iip::Symbol, expr::Expr, u0::Union{Symbol, Nothing}=nothing)
     @assert expr.head == :(=)
     x1, x2 = expr.args
     @assert x2.head == :call
-    f, x = x2.args
+    f, x... = x2.args
     define_expr = u0 === nothing ? :() : :($(x1) = similar($(u0)))
     return quote
         if $(esc(iip))
             $(esc(define_expr))
-            $(esc(f))($(esc(x1)), $(esc(x)))
+            $(esc(f))($(esc(x1)), $(esc.(x)...))
         else
             $(esc(expr))
         end
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 74566e8c3..6c5c3ce7f 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -43,11 +43,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
         error("Broyden currently only supports out-of-place nonlinear problems")
     end
 
-    atol = abstol !== nothing ? abstol :
-           (tc.abstol !== nothing ? tc.abstol :
-            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
-    rtol = reltol !== nothing ? reltol :
-           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
+    atol = _get_tolerance(abstol, tc.abstol, T)
+    rtol = _get_tolerance(reltol, tc.reltol, T)
 
     if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
         error("Broyden currently doesn't support SAFE_BEST termination modes")
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index b5e2b8200..d4bc77709 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -51,7 +51,7 @@ Computation, 75, 1429-1448.](https://www.researchgate.net/publication/220576479_
 - `max_inner_iterations`: the maximum number of iterations allowed for the inner loop of the
   algorithm. Used exclusively in `batched` mode. Defaults to `1000`.
 """
-struct SimpleDFSane{batched, T, TC} <: AbstractSimpleNonlinearSolveAlgorithm
+struct SimpleDFSane{T, TC} <: AbstractSimpleNonlinearSolveAlgorithm
     σ_min::T
     σ_max::T
     σ_1::T
@@ -62,47 +62,54 @@ struct SimpleDFSane{batched, T, TC} <: AbstractSimpleNonlinearSolveAlgorithm
     nexp::Int
     η_strategy::Function
     termination_condition::TC
-    max_inner_iterations::Int
+end
 
-    function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
-        M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
-        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2,
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing),
-        batched::Bool = false,
-        max_inner_iterations = 1000)
-        return new{batched, typeof(σ_min), typeof(termination_condition)}(σ_min,
-            σ_max,
-            σ_1,
+function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
+    M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+    nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2,
+    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+        abstol = nothing,
+        reltol = nothing),
+    batched::Bool = false,
+    max_inner_iterations = 1000)
+    if batched
+        return SimpleBatchedDFSane(; σₘᵢₙ = σ_min,
+            σₘₐₓ = σ_max,
+            σ₁ = σ_1,
             M,
             γ,
-            τ_min,
-            τ_max,
-            nexp,
-            η_strategy,
+            τₘᵢₙ = τ_min,
+            τₘₐₓ = τ_max,
+            nₑₓₚ = nexp,
+            ηₛ = η_strategy,
             termination_condition,
             max_inner_iterations)
     end
+    return SimpleDFSane{typeof(σ_min), typeof(termination_condition)}(σ_min,
+        σ_max,
+        σ_1,
+        M,
+        γ,
+        τ_min,
+        τ_max,
+        nexp,
+        η_strategy,
+        termination_condition)
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
     args...; abstol = nothing, reltol = nothing, maxiters = 1000,
-    kwargs...) where {batched}
+    kwargs...)
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
 
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
 
-    if batched
-        batch_size = size(x, 2)
-    end
-
     T = eltype(x)
     σ_min = float(alg.σ_min)
     σ_max = float(alg.σ_max)
-    σ_k = batched ? fill(float(alg.σ_1), 1, batch_size) : float(alg.σ_1)
+    σ_k = float(alg.σ_1)
 
     M = alg.M
     γ = float(alg.γ)
@@ -111,17 +118,12 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
     nexp = alg.nexp
     η_strategy = alg.η_strategy
 
-    batched && @assert ndims(x)==2 "Batched SimpleDFSane only supports 2D arrays"
-
     if SciMLBase.isinplace(prob)
         error("SimpleDFSane currently only supports out-of-place nonlinear problems")
     end
 
-    atol = abstol !== nothing ? abstol :
-           (tc.abstol !== nothing ? tc.abstol :
-            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
-    rtol = reltol !== nothing ? reltol :
-           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
+    atol = _get_tolerance(abstol, tc.abstol, T)
+    rtol = _get_tolerance(reltol, tc.reltol, T)
 
     if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
         error("SimpleDFSane currently doesn't support SAFE_BEST termination modes")
@@ -133,22 +135,12 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
 
     function ff(x)
         F = f(x)
-        f_k = if batched
-            sum(abs2, F; dims = 1) .^ (nexp / 2)
-        else
-            norm(F)^nexp
-        end
+        f_k = norm(F)^nexp
         return f_k, F
     end
 
     function generate_history(f_k, M)
-        if batched
-            history = similar(f_k, (M, length(f_k)))
-            history .= reshape(f_k, 1, :)
-            return history
-        else
-            return fill(f_k, M)
-        end
+        return fill(f_k, M)
     end
 
     f_k, F_k = ff(x)
@@ -158,17 +150,13 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
 
     for k in 1:maxiters
         # Spectral parameter range check
-        if batched
-            @. σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
-        else
-            σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
-        end
+        σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
 
         # Line search direction
         d = -σ_k .* F_k
 
         η = η_strategy(f_1, k, x, F_k)
-        f̄ = batched ? maximum(history_f_k; dims = 1) : maximum(history_f_k)
+        f̄ = maximum(history_f_k)
         α_p = α_1
         α_m = α_1
         x_new = @. x + α_p * d
@@ -179,38 +167,20 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
         while true
             inner_iterations += 1
 
-            if batched
-                criteria = @. f̄ + η - γ * α_p^2 * f_k
-                # NOTE: This is simply a heuristic, ideally we check using `all` but that is
-                #       typically very expensive for large problems
-                (sum(f_new .≤ criteria) ≥ batch_size ÷ 2) && break
-            else
-                criteria = f̄ + η - γ * α_p^2 * f_k
-                f_new ≤ criteria && break
-            end
+            criteria = f̄ + η - γ * α_p^2 * f_k
+            f_new ≤ criteria && break
 
             α_tp = @. α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
             x_new = @. x - α_m * d
             f_new, F_new = ff(x_new)
 
-            if batched
-                # NOTE: This is simply a heuristic, ideally we check using `all` but that is
-                #       typically very expensive for large problems
-                (sum(f_new .≤ criteria) ≥ batch_size ÷ 2) && break
-            else
-                f_new ≤ criteria && break
-            end
+            f_new ≤ criteria && break
 
             α_tm = @. α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
             α_p = @. clamp(α_tp, τ_min * α_p, τ_max * α_p)
             α_m = @. clamp(α_tm, τ_min * α_m, τ_max * α_m)
             x_new = @. x + α_p * d
             f_new, F_new = ff(x_new)
-
-            # NOTE: The original algorithm runs till either condition is satisfied, however,
-            #       for most batched problems like neural networks we only care about
-            #       approximate convergence
-            batched && (inner_iterations ≥ alg.max_inner_iterations) && break
         end
 
         if termination_condition(F_new, x_new, x, atol, rtol)
@@ -225,11 +195,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
         s_k = @. x_new - x
         y_k = @. F_new - F_k
 
-        if batched
-            σ_k = sum(abs2, s_k; dims = 1) ./ (sum(s_k .* y_k; dims = 1) .+ T(1e-5))
-        else
-            σ_k = (s_k' * s_k) / (s_k' * y_k)
-        end
+        σ_k = (s_k' * s_k) / (s_k' * y_k)
 
         # Take step
         x = x_new
@@ -237,11 +203,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{batched},
         f_k = f_new
 
         # Store function value
-        if batched
-            history_f_k[k % M + 1, :] .= vec(f_new)
-        else
-            history_f_k[k % M + 1] = f_new
-        end
+        history_f_k[k % M + 1] = f_new
     end
     return SciMLBase.build_solution(prob, alg, x, F_k; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
new file mode 100644
index 000000000..280bf2a39
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -0,0 +1,12 @@
+[deps]
+AbstractDifferentiation = "c29ec348-61ec-40c8-8164-b8c60e9d9f3d"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index dd54527a5..4fe316b34 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,9 +1,5 @@
-using SimpleNonlinearSolve
-using StaticArrays
-using BenchmarkTools
-using DiffEqBase
-using LinearAlgebra
-using Test
+using SimpleNonlinearSolve, StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
+    NNlib, AbstractDifferentiation, LinearSolve
 
 const BATCHED_BROYDEN_SOLVERS = []
 const BROYDEN_SOLVERS = []
@@ -476,9 +472,6 @@ for options in list_of_options
     @test all(abs.(f(u, p)) .< 1e-10)
 end
 
-# Batched Broyden
-using NNlib
-
 f, u0 = (u, p) -> u .* u .- p, randn(1, 3)
 
 p = [2.0 1.0 5.0];
@@ -488,7 +481,7 @@ sol = solve(probN, Broyden(batched = true))
 
 @test abs.(sol.u) ≈ sqrt.(p)
 
-for alg in (BATCHED_BROYDEN_SOLVERS...,
+@testset "Batched Solver: $(nameof(typeof(alg)))" for alg in (BATCHED_BROYDEN_SOLVERS...,
     BATCHED_LBROYDEN_SOLVERS...,
     BATCHED_DFSANE_SOLVERS...)
     sol = solve(probN, alg; abstol = 1e-3, reltol = 1e-3)

From 52b5bacf974cd41ed37ef292ba7edd4431d65411 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 27 Jun 2023 17:01:35 -0400
Subject: [PATCH 138/375] Remove docstrings to ensure users don't directly
 construct the batched solvers

---
 lib/SimpleNonlinearSolve/src/batched/broyden.jl | 14 --------------
 lib/SimpleNonlinearSolve/src/batched/raphson.jl | 16 ----------------
 2 files changed, 30 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/batched/broyden.jl b/lib/SimpleNonlinearSolve/src/batched/broyden.jl
index a301c4e14..85754880f 100644
--- a/lib/SimpleNonlinearSolve/src/batched/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/broyden.jl
@@ -1,17 +1,3 @@
-"""
-    BatchedBroyden(;
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing))
-
-A low-overhead batched implementation of Broyden capable of solving multiple nonlinear
-problems simultaneously.
-
-!!! note
-
-    To use this version, remember to load `NNlib`, i.e., `using NNlib` or
-    `import NNlib` must be present in your code.
-"""
 struct BatchedBroyden{TC <: NLSolveTerminationCondition} <:
     AbstractBatchedNonlinearSolveAlgorithm
     termination_condition::TC
diff --git a/lib/SimpleNonlinearSolve/src/batched/raphson.jl b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
index 0a28e338f..2be565fc5 100644
--- a/lib/SimpleNonlinearSolve/src/batched/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
@@ -1,19 +1,3 @@
-"""
-    SimpleBatchedNewtonRaphson(; chunk_size = Val{0}(),
-        autodiff = Val{true}(),
-        diff_type = Val{:forward},
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing))
-
-A low-overhead batched implementation of Newton-Raphson capable of solving multiple
-nonlinear problems simultaneously.
-
-!!! note
-
-    To use the `batched` version, remember to load `AbstractDifferentiation` and
-    `LinearSolve`.
-"""
 struct SimpleBatchedNewtonRaphson{AD, LS, TC <: NLSolveTerminationCondition} <:
     AbstractBatchedNonlinearSolveAlgorithm
     autodiff::AD

From c98922be35229fb0e0835c9eecb3b948aaa60b54 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 27 Jun 2023 18:38:35 -0400
Subject: [PATCH 139/375] LBroyden

---
 .../SimpleNonlinearSolveADLinearSolveExt.jl   |  25 +--
 .../ext/SimpleNonlinearSolveNNlibExt.jl       | 122 +++++++++++--
 .../src/batched/dfsane.jl                     |   2 +-
 .../src/batched/lbroyden.jl                   |   7 +
 lib/SimpleNonlinearSolve/src/broyden.jl       |  10 +-
 lib/SimpleNonlinearSolve/src/lbroyden.jl      | 163 ++++++++----------
 6 files changed, 210 insertions(+), 119 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
index 96ba9168a..d0a97eb86 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
@@ -1,8 +1,10 @@
 module SimpleNonlinearSolveADLinearSolveExt
 
-using AbstractDifferentiation, ArrayInterface, DiffEqBase, LinearAlgebra, LinearSolve,
+using AbstractDifferentiation,
+    ArrayInterface, DiffEqBase, LinearAlgebra, LinearSolve,
     SimpleNonlinearSolve, SciMLBase
-import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage, _result_from_storage, _get_tolerance, @maybeinplace
+import SimpleNonlinearSolve: _construct_batched_problem_structure,
+    _get_storage, _result_from_storage, _get_tolerance, @maybeinplace
 
 const AD = AbstractDifferentiation
 
@@ -20,19 +22,18 @@ function SimpleNonlinearSolve.SimpleBatchedNewtonRaphson(; chunk_size = Val{0}()
     # TODO: Use `diff_type`. FiniteDiff.jl is currently not available in AD.jl
     chunksize = SciMLBase._unwrap_val(chunk_size) == 0 ? nothing : chunk_size
     ad = SciMLBase._unwrap_val(autodiff) ?
-        AD.ForwardDiffBackend(; chunksize) :
-        AD.FiniteDifferencesBackend()
-    return SimpleBatchedNewtonRaphson{typeof(ad), Nothing, typeof(termination_condition)}(
-        ad,
+         AD.ForwardDiffBackend(; chunksize) :
+         AD.FiniteDifferencesBackend()
+    return SimpleBatchedNewtonRaphson{typeof(ad), Nothing, typeof(termination_condition)}(ad,
         nothing,
         termination_condition)
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
     alg::SimpleBatchedNewtonRaphson;
-    abstol=nothing,
-    reltol=nothing,
-    maxiters=1000,
+    abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000,
     kwargs...)
     iip = isinplace(prob)
     @assert !iip "SimpleBatchedNewtonRaphson currently only supports out-of-place nonlinear problems."
@@ -57,9 +58,9 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             alg,
             reconstruct(xₙ),
             reconstruct(fₙ);
-            retcode=ReturnCode.Success)
+            retcode = ReturnCode.Success)
 
-        solve(LinearProblem(𝓙, vec(fₙ); u0=vec(δx)), alg.linsolve; kwargs...)
+        solve(LinearProblem(𝓙, vec(fₙ); u0 = vec(δx)), alg.linsolve; kwargs...)
         xₙ .-= δx
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
@@ -83,7 +84,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         alg,
         reconstruct(xₙ),
         reconstruct(fₙ);
-        retcode=ReturnCode.MaxIters)
+        retcode = ReturnCode.MaxIters)
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
index e62e2bda7..c0faefd9b 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
@@ -1,18 +1,20 @@
 module SimpleNonlinearSolveNNlibExt
 
 using ArrayInterface, DiffEqBase, LinearAlgebra, NNlib, SimpleNonlinearSolve, SciMLBase
-import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage, _init_𝓙, _result_from_storage, _get_tolerance, @maybeinplace
+import SimpleNonlinearSolve: _construct_batched_problem_structure,
+    _get_storage, _init_𝓙, _result_from_storage, _get_tolerance, @maybeinplace
 
 function __init__()
     SimpleNonlinearSolve.NNlibExtLoaded[] = true
     return
 end
 
+# Broyden's method
 @views function SciMLBase.__solve(prob::NonlinearProblem,
     alg::BatchedBroyden;
-    abstol=nothing,
-    reltol=nothing,
-    maxiters=1000,
+    abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000,
     kwargs...)
     iip = isinplace(prob)
 
@@ -24,7 +26,7 @@ end
 
     storage = _get_storage(mode, u)
 
-    xₙ, xₙ₋₁, δx, δf = ntuple(_ -> copy(u), 4)
+    xₙ, xₙ₋₁, δxₙ, δf = ntuple(_ -> copy(u), 4)
     T = eltype(u)
 
     atol = _get_tolerance(abstol, tc.abstol, T)
@@ -41,16 +43,16 @@ end
         xₙ .= xₙ₋₁ .- 𝓙⁻¹f
 
         @maybeinplace iip fₙ=f(xₙ)
-        δx .= xₙ .- xₙ₋₁
+        δxₙ .= xₙ .- xₙ₋₁
         δf .= fₙ .- fₙ₋₁
 
         batched_mul!(reshape(𝓙⁻¹f, L, 1, N), 𝓙⁻¹, reshape(δf, L, 1, N))
-        δxᵀ = reshape(δx, 1, L, N)
+        δxₙᵀ = reshape(δxₙ, 1, L, N)
 
-        batched_mul!(reshape(xᵀ𝓙⁻¹δf, 1, 1, N), δxᵀ, reshape(𝓙⁻¹f, L, 1, N))
-        batched_mul!(xᵀ𝓙⁻¹, δxᵀ, 𝓙⁻¹)
-        δx .= (δx .- 𝓙⁻¹f) ./ (xᵀ𝓙⁻¹δf .+ T(1e-5))
-        batched_mul!(𝓙⁻¹, reshape(δx, L, 1, N), xᵀ𝓙⁻¹, one(T), one(T))
+        batched_mul!(reshape(xᵀ𝓙⁻¹δf, 1, 1, N), δxₙᵀ, reshape(𝓙⁻¹f, L, 1, N))
+        batched_mul!(xᵀ𝓙⁻¹, δxₙᵀ, 𝓙⁻¹)
+        δxₙ .= (δxₙ .- 𝓙⁻¹f) ./ (xᵀ𝓙⁻¹δf .+ T(1e-5))
+        batched_mul!(𝓙⁻¹, reshape(δxₙ, L, 1, N), xᵀ𝓙⁻¹, one(T), one(T))
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
             retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
@@ -74,7 +76,103 @@ end
         alg,
         reconstruct(xₙ),
         reconstruct(fₙ);
-        retcode=ReturnCode.MaxIters)
+        retcode = ReturnCode.MaxIters)
+end
+
+# Limited Memory Broyden's method
+@views function SciMLBase.__solve(prob::NonlinearProblem,
+    alg::BatchedLBroyden;
+    abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000,
+    kwargs...)
+    iip = isinplace(prob)
+
+    u, f, reconstruct = _construct_batched_problem_structure(prob)
+    L, N = size(u)
+    T = eltype(u)
+
+    tc = alg.termination_condition
+    mode = DiffEqBase.get_termination_mode(tc)
+
+    storage = _get_storage(mode, u)
+
+    η = min(maxiters, alg.threshold)
+    U = fill!(similar(u, (η, L, N)), zero(T))
+    Vᵀ = fill!(similar(u, (L, η, N)), zero(T))
+
+    xₙ, xₙ₋₁, δfₙ = ntuple(_ -> copy(u), 3)
+
+    atol = _get_tolerance(abstol, tc.abstol, T)
+    rtol = _get_tolerance(reltol, tc.reltol, T)
+    termination_condition = tc(storage)
+
+    @maybeinplace iip fₙ₋₁=f(xₙ) u
+    iip && (fₙ = copy(fₙ₋₁))
+    δxₙ = -copy(fₙ₋₁)
+    ηNx = similar(xₙ, η, N)
+
+    for i in 1:maxiters
+        @. xₙ = xₙ₋₁ - δxₙ
+        @maybeinplace iip fₙ=f(xₙ)
+        @. δxₙ = xₙ - xₙ₋₁
+        @. δfₙ = fₙ - fₙ₋₁
+
+        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
+            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
+            return DiffEqBase.build_solution(prob,
+                alg,
+                reconstruct(xₙ),
+                reconstruct(fₙ);
+                retcode)
+        end
+
+        _L = min(i, η)
+        _U = U[1:_L, :, :]
+        _Vᵀ = Vᵀ[:, 1:_L, :]
+
+        idx = mod1(i, η)
+
+        if i > 1
+            partial_ηNx = ηNx[1:_L, :]
+
+            _ηNx = reshape(partial_ηNx, 1, :, N)
+            batched_mul!(_ηNx, reshape(δxₙ, 1, L, N), _Vᵀ)
+            batched_mul!(Vᵀ[:, idx:idx, :], _ηNx, _U)
+            Vᵀ[:, idx, :] .-= δxₙ
+
+            _ηNx = reshape(partial_ηNx, :, 1, N)
+            batched_mul!(_ηNx, _U, reshape(δfₙ, L, 1, N))
+            batched_mul!(U[idx:idx, :, :], _Vᵀ, _ηNx)
+            U[idx, :, :] .-= δfₙ
+        else
+            Vᵀ[:, idx, :] .= -δxₙ
+            U[idx, :, :] .= -δfₙ
+        end
+
+        U[idx, :, :] .= (δxₙ .- U[idx, :, :]) ./
+                        (sum(Vᵀ[:, idx, :] .* δfₙ; dims = 1) .+
+                         convert(T, 1e-5))
+
+        _L = min(i + 1, η)
+        _ηNx = reshape(ηNx[1:_L, :], :, 1, N)
+        batched_mul!(_ηNx, U[1:_L, :, :], reshape(δfₙ, L, 1, N))
+        batched_mul!(reshape(δxₙ, L, 1, N), Vᵀ[:, 1:_L, :], _ηNx)
+
+        xₙ₋₁ .= xₙ
+        fₙ₋₁ .= fₙ
+    end
+
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        xₙ = storage.u
+        @maybeinplace iip fₙ=f(xₙ)
+    end
+
+    return DiffEqBase.build_solution(prob,
+        alg,
+        reconstruct(xₙ),
+        reconstruct(fₙ);
+        retcode = ReturnCode.MaxIters)
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
index fe7cbcddf..88f02ebe7 100644
--- a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
@@ -1,4 +1,4 @@
-@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition} <:
+Base.@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition} <:
               AbstractBatchedNonlinearSolveAlgorithm
     σₘᵢₙ::T = 1.0f-10
     σₘₐₓ::T = 1.0f+10
diff --git a/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl b/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
index e69de29bb..5934c8fe0 100644
--- a/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
@@ -0,0 +1,7 @@
+struct BatchedLBroyden{TC <: NLSolveTerminationCondition} <:
+    AbstractBatchedNonlinearSolveAlgorithm
+    termination_condition::TC
+    threshold::Int
+end
+
+# Implementation of solve using Package Extensions
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 6c5c3ce7f..adf94b033 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -30,6 +30,9 @@ end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
     abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
+    if SciMLBase.isinplace(prob)
+        error("Broyden currently only supports out-of-place nonlinear problems")
+    end
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
     f = Base.Fix2(prob.f, prob.p)
@@ -39,10 +42,6 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
     T = eltype(x)
     J⁻¹ = init_J(x)
 
-    if SciMLBase.isinplace(prob)
-        error("Broyden currently only supports out-of-place nonlinear problems")
-    end
-
     atol = _get_tolerance(abstol, tc.abstol, T)
     rtol = _get_tolerance(reltol, tc.reltol, T)
 
@@ -50,8 +49,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
         error("Broyden currently doesn't support SAFE_BEST termination modes")
     end
 
-    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
-              nothing
+    storage = _get_storage(mode, x)
     termination_condition = tc(storage)
 
     xₙ = x
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
index fc2b51a88..95ec3895e 100644
--- a/lib/SimpleNonlinearSolve/src/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -11,134 +11,121 @@ Broyden's method.
 
     This method is not very stable and can diverge even for very simple problems. This has mostly been
     tested for neural networks in DeepEquilibriumNetworks.jl.
+
+!!! note
+
+    To use the `batched` version, remember to load `NNlib`, i.e., `using NNlib` or
+    `import NNlib` must be present in your code.
 """
 struct LBroyden{batched, TC <: NLSolveTerminationCondition} <:
        AbstractSimpleNonlinearSolveAlgorithm
     termination_condition::TC
     threshold::Int
+end
 
-    function LBroyden(; batched = false, threshold::Int = 27,
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing))
-        return new{batched, typeof(termination_condition)}(termination_condition, threshold)
+function LBroyden(; batched = false, threshold::Int = 27,
+    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+        abstol = nothing,
+        reltol = nothing))
+    if batched
+        @assert NNlibExtLoaded[] "Please install and load `NNlib.jl` to use batched Broyden."
+        return BatchedLBroyden(termination_condition, threshold)
     end
+    return LBroyden{true, typeof(termination_condition)}(termination_condition, threshold)
 end
 
-@views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden{batched}, args...;
+@views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden, args...;
     abstol = nothing, reltol = nothing, maxiters = 1000,
-    kwargs...) where {batched}
+    kwargs...)
+    if SciMLBase.isinplace(prob)
+        error("LBroyden currently only supports out-of-place nonlinear problems")
+    end
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
-    threshold = min(maxiters, alg.threshold)
+    η = min(maxiters, alg.threshold)
     x = float(prob.u0)
 
-    batched && @assert ndims(x)==2 "Batched LBroyden only supports 2D arrays"
-
+    # FIXME: The scalar case currently is very inefficient
     if x isa Number
         restore_scalar = true
         x = [x]
-        f = u -> prob.f(u[], prob.p)
+        f = u -> [prob.f(u[], prob.p)]
     else
         f = Base.Fix2(prob.f, prob.p)
         restore_scalar = false
     end
 
-    fₙ = f(x)
+    L = length(x)
     T = eltype(x)
 
-    if SciMLBase.isinplace(prob)
-        error("LBroyden currently only supports out-of-place nonlinear problems")
-    end
-
-    U, Vᵀ = _init_lbroyden_state(batched, x, threshold)
+    U = fill!(similar(x, (η, L)), zero(T))
+    Vᵀ = fill!(similar(x, (L, η)), zero(T))
 
-    atol = abstol !== nothing ? abstol :
-           (tc.abstol !== nothing ? tc.abstol :
-            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
-    rtol = reltol !== nothing ? reltol :
-           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
-
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        error("LBroyden currently doesn't support SAFE_BEST termination modes")
-    end
-
-    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
-              nothing
+    atol = _get_tolerance(abstol, tc.abstol, T)
+    rtol = _get_tolerance(reltol, tc.reltol, T)
+    storage = _get_storage(mode, x)
     termination_condition = tc(storage)
 
-    xₙ = x
-    xₙ₋₁ = x
-    fₙ₋₁ = fₙ
-    update = fₙ
+    xₙ, xₙ₋₁, δfₙ = ntuple(_ -> copy(x), 3)
+    fₙ₋₁ = f(x)
+    δxₙ = -copy(fₙ₋₁)
+    ηNx = similar(xₙ, η)
+
     for i in 1:maxiters
-        xₙ = xₙ₋₁ .+ update
+        @. xₙ = xₙ₋₁ - δxₙ
         fₙ = f(xₙ)
-        Δxₙ = xₙ .- xₙ₋₁
-        Δfₙ = fₙ .- fₙ₋₁
+        @. δxₙ = xₙ - xₙ₋₁
+        @. δfₙ = fₙ - fₙ₋₁
 
-        if termination_condition(restore_scalar ? [fₙ] : fₙ, xₙ, xₙ₋₁, atol, rtol)
+        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
+            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, Val(false))
             xₙ = restore_scalar ? xₙ[] : xₙ
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
+            fₙ = restore_scalar ? fₙ[] : fₙ
+            return DiffEqBase.build_solution(prob, alg, xₙ, fₙ; retcode)
         end
 
-        _U = selectdim(U, 1, 1:min(threshold, i))
-        _Vᵀ = selectdim(Vᵀ, 2, 1:min(threshold, i))
-
-        vᵀ = _rmatvec(_U, _Vᵀ, Δxₙ)
-        mvec = _matvec(_U, _Vᵀ, Δfₙ)
-        u = (Δxₙ .- mvec) ./ (sum(vᵀ .* Δfₙ) .+ convert(T, 1e-5))
+        _L = min(i, η)
+        _U = U[1:_L, :]
+        _Vᵀ = Vᵀ[:, 1:_L]
 
-        selectdim(Vᵀ, 2, mod1(i, threshold)) .= vᵀ
-        selectdim(U, 1, mod1(i, threshold)) .= u
+        idx = mod1(i, η)
 
-        update = -_matvec(selectdim(U, 1, 1:min(threshold, i + 1)),
-            selectdim(Vᵀ, 2, 1:min(threshold, i + 1)), fₙ)
+        partial_ηNx = ηNx[1:_L]
 
-        xₙ₋₁ = xₙ
-        fₙ₋₁ = fₙ
-    end
+        if i > 1
+            _ηNx = reshape(partial_ηNx, 1, :)
+            mul!(_ηNx, reshape(δxₙ, 1, L), _Vᵀ)
+            mul!(Vᵀ[:, idx:idx], _ηNx, _U)
+            Vᵀ[:, idx] .-= δxₙ
 
-    xₙ = restore_scalar ? xₙ[] : xₙ
-    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
-end
+            _ηNx = reshape(partial_ηNx, :, 1)
+            mul!(_ηNx, _U, reshape(δfₙ, L, 1))
+            mul!(U[idx:idx, :], _Vᵀ, _ηNx)
+            U[idx, :] .-= δfₙ
+        else
+            Vᵀ[:, idx] .= -δxₙ
+            U[idx, :] .= -δfₙ
+        end
 
-function _init_lbroyden_state(batched::Bool, x, threshold)
-    T = eltype(x)
-    if batched
-        U = fill!(similar(x, (threshold, size(x, 1), size(x, 2))), zero(T))
-        Vᵀ = fill!(similar(x, (size(x, 1), threshold, size(x, 2))), zero(T))
-    else
-        U = fill!(similar(x, (threshold, length(x))), zero(T))
-        Vᵀ = fill!(similar(x, (length(x), threshold)), zero(T))
-    end
-    return U, Vᵀ
-end
+        U[idx, :] .= (δxₙ .- U[idx, :]) ./
+                     (sum(Vᵀ[:, idx] .* δfₙ) .+
+                      convert(T, 1e-5))
 
-function _rmatvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
-    x::Union{<:AbstractVector, <:Number})
-    length(U) == 0 && return x
-    return -x .+ vec((x' * Vᵀ) * U)
-end
+        _L = min(i + 1, η)
+        _ηNx = reshape(ηNx[1:_L], :, 1)
+        mul!(_ηNx, U[1:_L, :], reshape(δfₙ, L, 1))
+        mul!(reshape(δxₙ, L, 1), Vᵀ[:, 1:_L], _ηNx)
 
-function _rmatvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
-    x::AbstractMatrix) where {T1, T2}
-    length(U) == 0 && return x
-    Vᵀx = sum(Vᵀ .* reshape(x, size(x, 1), 1, size(x, 2)); dims = 1)
-    return -x .+ _drdims_sum(U .* permutedims(Vᵀx, (2, 1, 3)); dims = 1)
-end
+        xₙ₋₁ .= xₙ
+        fₙ₋₁ .= fₙ
+    end
 
-function _matvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
-    x::Union{<:AbstractVector, <:Number})
-    length(U) == 0 && return x
-    return -x .+ vec(Vᵀ * (U * x))
-end
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        xₙ = storage.u
+        fₙ = f(xₙ)
+    end
 
-function _matvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
-    x::AbstractMatrix) where {T1, T2}
-    length(U) == 0 && return x
-    xUᵀ = sum(reshape(x, size(x, 1), 1, size(x, 2)) .* permutedims(U, (2, 1, 3)); dims = 1)
-    return -x .+ _drdims_sum(xUᵀ .* Vᵀ; dims = 2)
+    xₙ = restore_scalar ? xₙ[] : xₙ
+    fₙ = restore_scalar ? fₙ[] : fₙ
+    return DiffEqBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
 end
-
-_drdims_sum(args...; dims = :) = dropdims(sum(args...; dims); dims)

From 48b84df7c8ac470a691c63ecaa4f9ca2d5f0178f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 28 Jun 2023 11:31:09 -0400
Subject: [PATCH 140/375] Revert "LBroyden"

This reverts commit c98922be35229fb0e0835c9eecb3b948aaa60b54.
---
 .../SimpleNonlinearSolveADLinearSolveExt.jl   |  25 ++-
 .../ext/SimpleNonlinearSolveNNlibExt.jl       | 122 ++-----------
 .../src/batched/dfsane.jl                     |   2 +-
 .../src/batched/lbroyden.jl                   |   7 -
 lib/SimpleNonlinearSolve/src/broyden.jl       |  10 +-
 lib/SimpleNonlinearSolve/src/lbroyden.jl      | 163 ++++++++++--------
 6 files changed, 119 insertions(+), 210 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
index d0a97eb86..96ba9168a 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
@@ -1,10 +1,8 @@
 module SimpleNonlinearSolveADLinearSolveExt
 
-using AbstractDifferentiation,
-    ArrayInterface, DiffEqBase, LinearAlgebra, LinearSolve,
+using AbstractDifferentiation, ArrayInterface, DiffEqBase, LinearAlgebra, LinearSolve,
     SimpleNonlinearSolve, SciMLBase
-import SimpleNonlinearSolve: _construct_batched_problem_structure,
-    _get_storage, _result_from_storage, _get_tolerance, @maybeinplace
+import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage, _result_from_storage, _get_tolerance, @maybeinplace
 
 const AD = AbstractDifferentiation
 
@@ -22,18 +20,19 @@ function SimpleNonlinearSolve.SimpleBatchedNewtonRaphson(; chunk_size = Val{0}()
     # TODO: Use `diff_type`. FiniteDiff.jl is currently not available in AD.jl
     chunksize = SciMLBase._unwrap_val(chunk_size) == 0 ? nothing : chunk_size
     ad = SciMLBase._unwrap_val(autodiff) ?
-         AD.ForwardDiffBackend(; chunksize) :
-         AD.FiniteDifferencesBackend()
-    return SimpleBatchedNewtonRaphson{typeof(ad), Nothing, typeof(termination_condition)}(ad,
+        AD.ForwardDiffBackend(; chunksize) :
+        AD.FiniteDifferencesBackend()
+    return SimpleBatchedNewtonRaphson{typeof(ad), Nothing, typeof(termination_condition)}(
+        ad,
         nothing,
         termination_condition)
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
     alg::SimpleBatchedNewtonRaphson;
-    abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000,
+    abstol=nothing,
+    reltol=nothing,
+    maxiters=1000,
     kwargs...)
     iip = isinplace(prob)
     @assert !iip "SimpleBatchedNewtonRaphson currently only supports out-of-place nonlinear problems."
@@ -58,9 +57,9 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             alg,
             reconstruct(xₙ),
             reconstruct(fₙ);
-            retcode = ReturnCode.Success)
+            retcode=ReturnCode.Success)
 
-        solve(LinearProblem(𝓙, vec(fₙ); u0 = vec(δx)), alg.linsolve; kwargs...)
+        solve(LinearProblem(𝓙, vec(fₙ); u0=vec(δx)), alg.linsolve; kwargs...)
         xₙ .-= δx
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
@@ -84,7 +83,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         alg,
         reconstruct(xₙ),
         reconstruct(fₙ);
-        retcode = ReturnCode.MaxIters)
+        retcode=ReturnCode.MaxIters)
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
index c0faefd9b..e62e2bda7 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
@@ -1,20 +1,18 @@
 module SimpleNonlinearSolveNNlibExt
 
 using ArrayInterface, DiffEqBase, LinearAlgebra, NNlib, SimpleNonlinearSolve, SciMLBase
-import SimpleNonlinearSolve: _construct_batched_problem_structure,
-    _get_storage, _init_𝓙, _result_from_storage, _get_tolerance, @maybeinplace
+import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage, _init_𝓙, _result_from_storage, _get_tolerance, @maybeinplace
 
 function __init__()
     SimpleNonlinearSolve.NNlibExtLoaded[] = true
     return
 end
 
-# Broyden's method
 @views function SciMLBase.__solve(prob::NonlinearProblem,
     alg::BatchedBroyden;
-    abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000,
+    abstol=nothing,
+    reltol=nothing,
+    maxiters=1000,
     kwargs...)
     iip = isinplace(prob)
 
@@ -26,7 +24,7 @@ end
 
     storage = _get_storage(mode, u)
 
-    xₙ, xₙ₋₁, δxₙ, δf = ntuple(_ -> copy(u), 4)
+    xₙ, xₙ₋₁, δx, δf = ntuple(_ -> copy(u), 4)
     T = eltype(u)
 
     atol = _get_tolerance(abstol, tc.abstol, T)
@@ -43,16 +41,16 @@ end
         xₙ .= xₙ₋₁ .- 𝓙⁻¹f
 
         @maybeinplace iip fₙ=f(xₙ)
-        δxₙ .= xₙ .- xₙ₋₁
+        δx .= xₙ .- xₙ₋₁
         δf .= fₙ .- fₙ₋₁
 
         batched_mul!(reshape(𝓙⁻¹f, L, 1, N), 𝓙⁻¹, reshape(δf, L, 1, N))
-        δxₙᵀ = reshape(δxₙ, 1, L, N)
+        δxᵀ = reshape(δx, 1, L, N)
 
-        batched_mul!(reshape(xᵀ𝓙⁻¹δf, 1, 1, N), δxₙᵀ, reshape(𝓙⁻¹f, L, 1, N))
-        batched_mul!(xᵀ𝓙⁻¹, δxₙᵀ, 𝓙⁻¹)
-        δxₙ .= (δxₙ .- 𝓙⁻¹f) ./ (xᵀ𝓙⁻¹δf .+ T(1e-5))
-        batched_mul!(𝓙⁻¹, reshape(δxₙ, L, 1, N), xᵀ𝓙⁻¹, one(T), one(T))
+        batched_mul!(reshape(xᵀ𝓙⁻¹δf, 1, 1, N), δxᵀ, reshape(𝓙⁻¹f, L, 1, N))
+        batched_mul!(xᵀ𝓙⁻¹, δxᵀ, 𝓙⁻¹)
+        δx .= (δx .- 𝓙⁻¹f) ./ (xᵀ𝓙⁻¹δf .+ T(1e-5))
+        batched_mul!(𝓙⁻¹, reshape(δx, L, 1, N), xᵀ𝓙⁻¹, one(T), one(T))
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
             retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
@@ -76,103 +74,7 @@ end
         alg,
         reconstruct(xₙ),
         reconstruct(fₙ);
-        retcode = ReturnCode.MaxIters)
-end
-
-# Limited Memory Broyden's method
-@views function SciMLBase.__solve(prob::NonlinearProblem,
-    alg::BatchedLBroyden;
-    abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000,
-    kwargs...)
-    iip = isinplace(prob)
-
-    u, f, reconstruct = _construct_batched_problem_structure(prob)
-    L, N = size(u)
-    T = eltype(u)
-
-    tc = alg.termination_condition
-    mode = DiffEqBase.get_termination_mode(tc)
-
-    storage = _get_storage(mode, u)
-
-    η = min(maxiters, alg.threshold)
-    U = fill!(similar(u, (η, L, N)), zero(T))
-    Vᵀ = fill!(similar(u, (L, η, N)), zero(T))
-
-    xₙ, xₙ₋₁, δfₙ = ntuple(_ -> copy(u), 3)
-
-    atol = _get_tolerance(abstol, tc.abstol, T)
-    rtol = _get_tolerance(reltol, tc.reltol, T)
-    termination_condition = tc(storage)
-
-    @maybeinplace iip fₙ₋₁=f(xₙ) u
-    iip && (fₙ = copy(fₙ₋₁))
-    δxₙ = -copy(fₙ₋₁)
-    ηNx = similar(xₙ, η, N)
-
-    for i in 1:maxiters
-        @. xₙ = xₙ₋₁ - δxₙ
-        @maybeinplace iip fₙ=f(xₙ)
-        @. δxₙ = xₙ - xₙ₋₁
-        @. δfₙ = fₙ - fₙ₋₁
-
-        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
-            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
-            return DiffEqBase.build_solution(prob,
-                alg,
-                reconstruct(xₙ),
-                reconstruct(fₙ);
-                retcode)
-        end
-
-        _L = min(i, η)
-        _U = U[1:_L, :, :]
-        _Vᵀ = Vᵀ[:, 1:_L, :]
-
-        idx = mod1(i, η)
-
-        if i > 1
-            partial_ηNx = ηNx[1:_L, :]
-
-            _ηNx = reshape(partial_ηNx, 1, :, N)
-            batched_mul!(_ηNx, reshape(δxₙ, 1, L, N), _Vᵀ)
-            batched_mul!(Vᵀ[:, idx:idx, :], _ηNx, _U)
-            Vᵀ[:, idx, :] .-= δxₙ
-
-            _ηNx = reshape(partial_ηNx, :, 1, N)
-            batched_mul!(_ηNx, _U, reshape(δfₙ, L, 1, N))
-            batched_mul!(U[idx:idx, :, :], _Vᵀ, _ηNx)
-            U[idx, :, :] .-= δfₙ
-        else
-            Vᵀ[:, idx, :] .= -δxₙ
-            U[idx, :, :] .= -δfₙ
-        end
-
-        U[idx, :, :] .= (δxₙ .- U[idx, :, :]) ./
-                        (sum(Vᵀ[:, idx, :] .* δfₙ; dims = 1) .+
-                         convert(T, 1e-5))
-
-        _L = min(i + 1, η)
-        _ηNx = reshape(ηNx[1:_L, :], :, 1, N)
-        batched_mul!(_ηNx, U[1:_L, :, :], reshape(δfₙ, L, 1, N))
-        batched_mul!(reshape(δxₙ, L, 1, N), Vᵀ[:, 1:_L, :], _ηNx)
-
-        xₙ₋₁ .= xₙ
-        fₙ₋₁ .= fₙ
-    end
-
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        xₙ = storage.u
-        @maybeinplace iip fₙ=f(xₙ)
-    end
-
-    return DiffEqBase.build_solution(prob,
-        alg,
-        reconstruct(xₙ),
-        reconstruct(fₙ);
-        retcode = ReturnCode.MaxIters)
+        retcode=ReturnCode.MaxIters)
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
index 88f02ebe7..fe7cbcddf 100644
--- a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
@@ -1,4 +1,4 @@
-Base.@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition} <:
+@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition} <:
               AbstractBatchedNonlinearSolveAlgorithm
     σₘᵢₙ::T = 1.0f-10
     σₘₐₓ::T = 1.0f+10
diff --git a/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl b/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
index 5934c8fe0..e69de29bb 100644
--- a/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
@@ -1,7 +0,0 @@
-struct BatchedLBroyden{TC <: NLSolveTerminationCondition} <:
-    AbstractBatchedNonlinearSolveAlgorithm
-    termination_condition::TC
-    threshold::Int
-end
-
-# Implementation of solve using Package Extensions
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index adf94b033..6c5c3ce7f 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -30,9 +30,6 @@ end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
     abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
-    if SciMLBase.isinplace(prob)
-        error("Broyden currently only supports out-of-place nonlinear problems")
-    end
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
     f = Base.Fix2(prob.f, prob.p)
@@ -42,6 +39,10 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
     T = eltype(x)
     J⁻¹ = init_J(x)
 
+    if SciMLBase.isinplace(prob)
+        error("Broyden currently only supports out-of-place nonlinear problems")
+    end
+
     atol = _get_tolerance(abstol, tc.abstol, T)
     rtol = _get_tolerance(reltol, tc.reltol, T)
 
@@ -49,7 +50,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
         error("Broyden currently doesn't support SAFE_BEST termination modes")
     end
 
-    storage = _get_storage(mode, x)
+    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
+              nothing
     termination_condition = tc(storage)
 
     xₙ = x
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
index 95ec3895e..fc2b51a88 100644
--- a/lib/SimpleNonlinearSolve/src/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -11,121 +11,134 @@ Broyden's method.
 
     This method is not very stable and can diverge even for very simple problems. This has mostly been
     tested for neural networks in DeepEquilibriumNetworks.jl.
-
-!!! note
-
-    To use the `batched` version, remember to load `NNlib`, i.e., `using NNlib` or
-    `import NNlib` must be present in your code.
 """
 struct LBroyden{batched, TC <: NLSolveTerminationCondition} <:
        AbstractSimpleNonlinearSolveAlgorithm
     termination_condition::TC
     threshold::Int
-end
 
-function LBroyden(; batched = false, threshold::Int = 27,
-    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-        abstol = nothing,
-        reltol = nothing))
-    if batched
-        @assert NNlibExtLoaded[] "Please install and load `NNlib.jl` to use batched Broyden."
-        return BatchedLBroyden(termination_condition, threshold)
+    function LBroyden(; batched = false, threshold::Int = 27,
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing))
+        return new{batched, typeof(termination_condition)}(termination_condition, threshold)
     end
-    return LBroyden{true, typeof(termination_condition)}(termination_condition, threshold)
 end
 
-@views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden, args...;
+@views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden{batched}, args...;
     abstol = nothing, reltol = nothing, maxiters = 1000,
-    kwargs...)
-    if SciMLBase.isinplace(prob)
-        error("LBroyden currently only supports out-of-place nonlinear problems")
-    end
+    kwargs...) where {batched}
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
-    η = min(maxiters, alg.threshold)
+    threshold = min(maxiters, alg.threshold)
     x = float(prob.u0)
 
-    # FIXME: The scalar case currently is very inefficient
+    batched && @assert ndims(x)==2 "Batched LBroyden only supports 2D arrays"
+
     if x isa Number
         restore_scalar = true
         x = [x]
-        f = u -> [prob.f(u[], prob.p)]
+        f = u -> prob.f(u[], prob.p)
     else
         f = Base.Fix2(prob.f, prob.p)
         restore_scalar = false
     end
 
-    L = length(x)
+    fₙ = f(x)
     T = eltype(x)
 
-    U = fill!(similar(x, (η, L)), zero(T))
-    Vᵀ = fill!(similar(x, (L, η)), zero(T))
+    if SciMLBase.isinplace(prob)
+        error("LBroyden currently only supports out-of-place nonlinear problems")
+    end
+
+    U, Vᵀ = _init_lbroyden_state(batched, x, threshold)
 
-    atol = _get_tolerance(abstol, tc.abstol, T)
-    rtol = _get_tolerance(reltol, tc.reltol, T)
-    storage = _get_storage(mode, x)
-    termination_condition = tc(storage)
+    atol = abstol !== nothing ? abstol :
+           (tc.abstol !== nothing ? tc.abstol :
+            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
+    rtol = reltol !== nothing ? reltol :
+           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
 
-    xₙ, xₙ₋₁, δfₙ = ntuple(_ -> copy(x), 3)
-    fₙ₋₁ = f(x)
-    δxₙ = -copy(fₙ₋₁)
-    ηNx = similar(xₙ, η)
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        error("LBroyden currently doesn't support SAFE_BEST termination modes")
+    end
+
+    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
+              nothing
+    termination_condition = tc(storage)
 
+    xₙ = x
+    xₙ₋₁ = x
+    fₙ₋₁ = fₙ
+    update = fₙ
     for i in 1:maxiters
-        @. xₙ = xₙ₋₁ - δxₙ
+        xₙ = xₙ₋₁ .+ update
         fₙ = f(xₙ)
-        @. δxₙ = xₙ - xₙ₋₁
-        @. δfₙ = fₙ - fₙ₋₁
+        Δxₙ = xₙ .- xₙ₋₁
+        Δfₙ = fₙ .- fₙ₋₁
 
-        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
-            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, Val(false))
+        if termination_condition(restore_scalar ? [fₙ] : fₙ, xₙ, xₙ₋₁, atol, rtol)
             xₙ = restore_scalar ? xₙ[] : xₙ
-            fₙ = restore_scalar ? fₙ[] : fₙ
-            return DiffEqBase.build_solution(prob, alg, xₙ, fₙ; retcode)
+            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
         end
 
-        _L = min(i, η)
-        _U = U[1:_L, :]
-        _Vᵀ = Vᵀ[:, 1:_L]
-
-        idx = mod1(i, η)
+        _U = selectdim(U, 1, 1:min(threshold, i))
+        _Vᵀ = selectdim(Vᵀ, 2, 1:min(threshold, i))
 
-        partial_ηNx = ηNx[1:_L]
+        vᵀ = _rmatvec(_U, _Vᵀ, Δxₙ)
+        mvec = _matvec(_U, _Vᵀ, Δfₙ)
+        u = (Δxₙ .- mvec) ./ (sum(vᵀ .* Δfₙ) .+ convert(T, 1e-5))
 
-        if i > 1
-            _ηNx = reshape(partial_ηNx, 1, :)
-            mul!(_ηNx, reshape(δxₙ, 1, L), _Vᵀ)
-            mul!(Vᵀ[:, idx:idx], _ηNx, _U)
-            Vᵀ[:, idx] .-= δxₙ
+        selectdim(Vᵀ, 2, mod1(i, threshold)) .= vᵀ
+        selectdim(U, 1, mod1(i, threshold)) .= u
 
-            _ηNx = reshape(partial_ηNx, :, 1)
-            mul!(_ηNx, _U, reshape(δfₙ, L, 1))
-            mul!(U[idx:idx, :], _Vᵀ, _ηNx)
-            U[idx, :] .-= δfₙ
-        else
-            Vᵀ[:, idx] .= -δxₙ
-            U[idx, :] .= -δfₙ
-        end
+        update = -_matvec(selectdim(U, 1, 1:min(threshold, i + 1)),
+            selectdim(Vᵀ, 2, 1:min(threshold, i + 1)), fₙ)
 
-        U[idx, :] .= (δxₙ .- U[idx, :]) ./
-                     (sum(Vᵀ[:, idx] .* δfₙ) .+
-                      convert(T, 1e-5))
+        xₙ₋₁ = xₙ
+        fₙ₋₁ = fₙ
+    end
 
-        _L = min(i + 1, η)
-        _ηNx = reshape(ηNx[1:_L], :, 1)
-        mul!(_ηNx, U[1:_L, :], reshape(δfₙ, L, 1))
-        mul!(reshape(δxₙ, L, 1), Vᵀ[:, 1:_L], _ηNx)
+    xₙ = restore_scalar ? xₙ[] : xₙ
+    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+end
 
-        xₙ₋₁ .= xₙ
-        fₙ₋₁ .= fₙ
+function _init_lbroyden_state(batched::Bool, x, threshold)
+    T = eltype(x)
+    if batched
+        U = fill!(similar(x, (threshold, size(x, 1), size(x, 2))), zero(T))
+        Vᵀ = fill!(similar(x, (size(x, 1), threshold, size(x, 2))), zero(T))
+    else
+        U = fill!(similar(x, (threshold, length(x))), zero(T))
+        Vᵀ = fill!(similar(x, (length(x), threshold)), zero(T))
     end
+    return U, Vᵀ
+end
 
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        xₙ = storage.u
-        fₙ = f(xₙ)
-    end
+function _rmatvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
+    x::Union{<:AbstractVector, <:Number})
+    length(U) == 0 && return x
+    return -x .+ vec((x' * Vᵀ) * U)
+end
 
-    xₙ = restore_scalar ? xₙ[] : xₙ
-    fₙ = restore_scalar ? fₙ[] : fₙ
-    return DiffEqBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+function _rmatvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
+    x::AbstractMatrix) where {T1, T2}
+    length(U) == 0 && return x
+    Vᵀx = sum(Vᵀ .* reshape(x, size(x, 1), 1, size(x, 2)); dims = 1)
+    return -x .+ _drdims_sum(U .* permutedims(Vᵀx, (2, 1, 3)); dims = 1)
 end
+
+function _matvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
+    x::Union{<:AbstractVector, <:Number})
+    length(U) == 0 && return x
+    return -x .+ vec(Vᵀ * (U * x))
+end
+
+function _matvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
+    x::AbstractMatrix) where {T1, T2}
+    length(U) == 0 && return x
+    xUᵀ = sum(reshape(x, size(x, 1), 1, size(x, 2)) .* permutedims(U, (2, 1, 3)); dims = 1)
+    return -x .+ _drdims_sum(xUᵀ .* Vᵀ; dims = 2)
+end
+
+_drdims_sum(args...; dims = :) = dropdims(sum(args...; dims); dims)

From 4455c08bfdad3f8c5118bc8a9138b9e95bda3545 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 28 Jun 2023 11:31:24 -0400
Subject: [PATCH 141/375] Format

---
 .../SimpleNonlinearSolveADLinearSolveExt.jl   | 25 ++++++++++---------
 .../ext/SimpleNonlinearSolveNNlibExt.jl       | 11 ++++----
 .../src/SimpleNonlinearSolve.jl               |  3 ++-
 .../src/batched/broyden.jl                    |  2 +-
 .../src/batched/dfsane.jl                     | 24 +++++++++---------
 .../src/batched/lbroyden.jl                   |  1 +
 .../src/batched/raphson.jl                    |  2 +-
 lib/SimpleNonlinearSolve/src/batched/utils.jl |  4 +--
 lib/SimpleNonlinearSolve/src/raphson.jl       | 10 ++++----
 9 files changed, 43 insertions(+), 39 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
index 96ba9168a..d0a97eb86 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
@@ -1,8 +1,10 @@
 module SimpleNonlinearSolveADLinearSolveExt
 
-using AbstractDifferentiation, ArrayInterface, DiffEqBase, LinearAlgebra, LinearSolve,
+using AbstractDifferentiation,
+    ArrayInterface, DiffEqBase, LinearAlgebra, LinearSolve,
     SimpleNonlinearSolve, SciMLBase
-import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage, _result_from_storage, _get_tolerance, @maybeinplace
+import SimpleNonlinearSolve: _construct_batched_problem_structure,
+    _get_storage, _result_from_storage, _get_tolerance, @maybeinplace
 
 const AD = AbstractDifferentiation
 
@@ -20,19 +22,18 @@ function SimpleNonlinearSolve.SimpleBatchedNewtonRaphson(; chunk_size = Val{0}()
     # TODO: Use `diff_type`. FiniteDiff.jl is currently not available in AD.jl
     chunksize = SciMLBase._unwrap_val(chunk_size) == 0 ? nothing : chunk_size
     ad = SciMLBase._unwrap_val(autodiff) ?
-        AD.ForwardDiffBackend(; chunksize) :
-        AD.FiniteDifferencesBackend()
-    return SimpleBatchedNewtonRaphson{typeof(ad), Nothing, typeof(termination_condition)}(
-        ad,
+         AD.ForwardDiffBackend(; chunksize) :
+         AD.FiniteDifferencesBackend()
+    return SimpleBatchedNewtonRaphson{typeof(ad), Nothing, typeof(termination_condition)}(ad,
         nothing,
         termination_condition)
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
     alg::SimpleBatchedNewtonRaphson;
-    abstol=nothing,
-    reltol=nothing,
-    maxiters=1000,
+    abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000,
     kwargs...)
     iip = isinplace(prob)
     @assert !iip "SimpleBatchedNewtonRaphson currently only supports out-of-place nonlinear problems."
@@ -57,9 +58,9 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             alg,
             reconstruct(xₙ),
             reconstruct(fₙ);
-            retcode=ReturnCode.Success)
+            retcode = ReturnCode.Success)
 
-        solve(LinearProblem(𝓙, vec(fₙ); u0=vec(δx)), alg.linsolve; kwargs...)
+        solve(LinearProblem(𝓙, vec(fₙ); u0 = vec(δx)), alg.linsolve; kwargs...)
         xₙ .-= δx
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
@@ -83,7 +84,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         alg,
         reconstruct(xₙ),
         reconstruct(fₙ);
-        retcode=ReturnCode.MaxIters)
+        retcode = ReturnCode.MaxIters)
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
index e62e2bda7..5b06530a6 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
@@ -1,7 +1,8 @@
 module SimpleNonlinearSolveNNlibExt
 
 using ArrayInterface, DiffEqBase, LinearAlgebra, NNlib, SimpleNonlinearSolve, SciMLBase
-import SimpleNonlinearSolve: _construct_batched_problem_structure, _get_storage, _init_𝓙, _result_from_storage, _get_tolerance, @maybeinplace
+import SimpleNonlinearSolve: _construct_batched_problem_structure,
+    _get_storage, _init_𝓙, _result_from_storage, _get_tolerance, @maybeinplace
 
 function __init__()
     SimpleNonlinearSolve.NNlibExtLoaded[] = true
@@ -10,9 +11,9 @@ end
 
 @views function SciMLBase.__solve(prob::NonlinearProblem,
     alg::BatchedBroyden;
-    abstol=nothing,
-    reltol=nothing,
-    maxiters=1000,
+    abstol = nothing,
+    reltol = nothing,
+    maxiters = 1000,
     kwargs...)
     iip = isinplace(prob)
 
@@ -74,7 +75,7 @@ end
         alg,
         reconstruct(xₙ),
         reconstruct(fₙ);
-        retcode=ReturnCode.MaxIters)
+        retcode = ReturnCode.MaxIters)
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 49aef9fc8..23b332fb3 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -22,7 +22,8 @@ abstract type AbstractSimpleNonlinearSolveAlgorithm <: SciMLBase.AbstractNonline
 abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 abstract type AbstractNewtonAlgorithm{CS, AD, FDT} <: AbstractSimpleNonlinearSolveAlgorithm end
 abstract type AbstractImmutableNonlinearSolver <: AbstractSimpleNonlinearSolveAlgorithm end
-abstract type AbstractBatchedNonlinearSolveAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
+abstract type AbstractBatchedNonlinearSolveAlgorithm <:
+              AbstractSimpleNonlinearSolveAlgorithm end
 
 include("utils.jl")
 include("bisection.jl")
diff --git a/lib/SimpleNonlinearSolve/src/batched/broyden.jl b/lib/SimpleNonlinearSolve/src/batched/broyden.jl
index 85754880f..ed3cd5dfc 100644
--- a/lib/SimpleNonlinearSolve/src/batched/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/broyden.jl
@@ -1,5 +1,5 @@
 struct BatchedBroyden{TC <: NLSolveTerminationCondition} <:
-    AbstractBatchedNonlinearSolveAlgorithm
+       AbstractBatchedNonlinearSolveAlgorithm
     termination_condition::TC
 end
 
diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
index fe7cbcddf..09fc37f8d 100644
--- a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
@@ -1,4 +1,4 @@
-@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition} <:
+Base.@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition} <:
               AbstractBatchedNonlinearSolveAlgorithm
     σₘᵢₙ::T = 1.0f-10
     σₘₐₓ::T = 1.0f+10
@@ -10,17 +10,17 @@
     nₑₓₚ::Int = 2
     ηₛ::F = (f₍ₙₒᵣₘ₎₁, n, xₙ, fₙ) -> f₍ₙₒᵣₘ₎₁ ./ n .^ 2
     termination_condition::TC = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-        abstol=nothing,
-        reltol=nothing)
+        abstol = nothing,
+        reltol = nothing)
     max_inner_iterations::Int = 1000
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
     alg::SimpleBatchedDFSane,
     args...;
-    abstol=nothing,
-    reltol=nothing,
-    maxiters=100,
+    abstol = nothing,
+    reltol = nothing,
+    maxiters = 100,
     kwargs...)
     iip = isinplace(prob)
 
@@ -60,7 +60,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         return fₓ
     end
 
-    @maybeinplace iip fₙ₋₁ = ff!(f₍ₙₒᵣₘ₎ₙ₋₁, xₙ) xₙ
+    @maybeinplace iip fₙ₋₁=ff!(f₍ₙₒᵣₘ₎ₙ₋₁, xₙ) xₙ
     iip && (fₙ = similar(fₙ₋₁))
     ℋ = repeat(f₍ₙₒᵣₘ₎ₙ₋₁, M, 1)
     f̄ = similar(ℋ, 1, N)
@@ -79,7 +79,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         fill!(α₋, α₁)
         @. xₙ = xₙ₋₁ + α₊ * 𝒹
 
-        @maybeinplace iip fₙ = ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
+        @maybeinplace iip fₙ=ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
 
         for _ in 1:(alg.max_inner_iterations)
             𝒸 = @. f̄ + η - γ * α₊^2 * f₍ₙₒᵣₘ₎ₙ₋₁
@@ -90,7 +90,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                 τₘᵢₙ * α₊,
                 τₘₐₓ * α₊)
             @. xₙ = xₙ₋₁ - α₋ * 𝒹
-            @maybeinplace iip fₙ = ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
+            @maybeinplace iip fₙ=ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
 
             (sum(f₍ₙₒᵣₘ₎ₙ .≤ 𝒸) ≥ N ÷ 2) && break
 
@@ -98,7 +98,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                 τₘᵢₙ * α₋,
                 τₘₐₓ * α₋)
             @. xₙ = xₙ₋₁ + α₊ * 𝒹
-            @maybeinplace iip fₙ = ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
+            @maybeinplace iip fₙ=ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
         end
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
@@ -129,12 +129,12 @@ function SciMLBase.__solve(prob::NonlinearProblem,
 
     if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
         xₙ = storage.u
-        @maybeinplace iip fₙ = f(xₙ)
+        @maybeinplace iip fₙ=f(xₙ)
     end
 
     return DiffEqBase.build_solution(prob,
         alg,
         reconstruct(xₙ),
         reconstruct(fₙ);
-        retcode=ReturnCode.MaxIters)
+        retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl b/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
index e69de29bb..8b1378917 100644
--- a/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
@@ -0,0 +1 @@
+
diff --git a/lib/SimpleNonlinearSolve/src/batched/raphson.jl b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
index 2be565fc5..63c03ec89 100644
--- a/lib/SimpleNonlinearSolve/src/batched/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
@@ -1,5 +1,5 @@
 struct SimpleBatchedNewtonRaphson{AD, LS, TC <: NLSolveTerminationCondition} <:
-    AbstractBatchedNonlinearSolveAlgorithm
+       AbstractBatchedNonlinearSolveAlgorithm
     autodiff::AD
     linsolve::LS
     termination_condition::TC
diff --git a/lib/SimpleNonlinearSolve/src/batched/utils.jl b/lib/SimpleNonlinearSolve/src/batched/utils.jl
index b5dbd597f..7b85011f1 100644
--- a/lib/SimpleNonlinearSolve/src/batched/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/utils.jl
@@ -1,4 +1,4 @@
-macro maybeinplace(iip::Symbol, expr::Expr, u0::Union{Symbol, Nothing}=nothing)
+macro maybeinplace(iip::Symbol, expr::Expr, u0::Union{Symbol, Nothing} = nothing)
     @assert expr.head == :(=)
     x1, x2 = expr.args
     @assert x2.head == :call
@@ -64,7 +64,7 @@ function _result_from_storage(storage::NLSolveSafeTerminationResult, xₙ, fₙ,
         return ReturnCode.Success, xₙ, fₙ
     else
         if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-            @maybeinplace iip fₙ = f(xₙ)
+            @maybeinplace iip fₙ=f(xₙ)
             return ReturnCode.Terminated, storage.u, fₙ
         else
             return ReturnCode.Terminated, xₙ, fₙ
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 59eb1ed6d..2621c587e 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -35,7 +35,7 @@ and static array problems.
 """
 struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT} end
 
-function SimpleNewtonRaphson(; batched=false,
+function SimpleNewtonRaphson(; batched = false,
     chunk_size = Val{0}(),
     autodiff = Val{true}(),
     diff_type = Val{:forward},
@@ -46,10 +46,10 @@ function SimpleNewtonRaphson(; batched=false,
     if batched
         @assert ADLinearSolveExtLoaded[] "Please install and load `LinearSolve.jl` and `AbstractDifferentiation.jl` to use batched Newton-Raphson."
         termination_condition = ismissing(termination_condition) ?
-            NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-                abstol = nothing,
-                reltol = nothing) :
-            termination_condition
+                                NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing) :
+                                termination_condition
         return SimpleBatchedNewtonRaphson(; chunk_size,
             autodiff,
             diff_type,

From cf90edd9cabdce2c1c7908e78986816f53187794 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Wed, 28 Jun 2023 21:31:34 +0200
Subject: [PATCH 142/375] final fixes

---
 lib/SimpleNonlinearSolve/src/itp.jl | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index bd5f888dd..b233c4f6b 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -13,13 +13,10 @@ struct Itp <: AbstractBracketingAlgorithm
     n0::Int
     function Itp(;k1::Real = 0.1, k2::Real = 2.0, n0::Int = 1)
         if k1 < 0
-            ArgumentError("Hyper-parameter κ₁ should not be negative")
-        end
-        if !isa(n0, Int)
-            ArgumentError("Hyper-parameter n₀ should be an Integer")
+            error("Hyper-parameter κ₁ should not be negative")
         end
         if n0 < 0
-            ArgumentError("Hyper-parameter n₀ should not be negative")
+            error("Hyper-parameter n₀ should not be negative")
         end
         if k2 < 1 || k2 > (1.5 + sqrt(5) / 2)
             ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where ϕ ≈ 1.618... is the golden ratio")
@@ -29,7 +26,7 @@ struct Itp <: AbstractBracketingAlgorithm
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
-                            args...; abstol = nothing,
+                            args...; abstol = 1.0e-8,
                             maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b
@@ -47,7 +44,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
     #defining variables/cache
     k1 = alg.k1
     k2 = alg.k2
-    n0 = alg.k3
+    n0 = alg.n0
     n_h = ceil(log2((right - left) / (2 * ϵ)))
     n_max = n_h + n0
     mid = (left + right) / 2

From b550942242cb0e9b39b31e92dc83de0c8a8296e6 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 29 Jun 2023 01:59:30 +0200
Subject: [PATCH 143/375] added itp tests

---
 lib/SimpleNonlinearSolve/src/itp.jl         |  4 ++--
 lib/SimpleNonlinearSolve/test/basictests.jl | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index b233c4f6b..c0d69f595 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -26,7 +26,7 @@ struct Itp <: AbstractBracketingAlgorithm
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
-                            args...; abstol = 1.0e-8,
+                            args...; abstol = 1.0e-10,
                             maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b
@@ -94,7 +94,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
         mid = (left + right) / 2
 
         if (right - left < 2 * ϵ)
-            return SciMLBase.build_solution(prob, alg, mid, fl;
+            return SciMLBase.build_solution(prob, alg, mid, f(mid);
             retcode = ReturnCode.Success, left = left,
             right = right)
         end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index aea173157..34ef7096f 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -215,6 +215,18 @@ for p in 1.1:0.1:100.0
     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
 end
 
+# ITP
+g = function (p)
+    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
+    sol = solve(probN, Itp())
+    return sol.u
+end
+
+for p in 1.1:0.1:100.0
+    @test g(p) ≈ sqrt(p)
+    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+end
+
 # Alefeld
 g = function (p)
     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
@@ -242,7 +254,7 @@ end
 f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
 t = (p) -> [sqrt(p[2] / p[1])]
 p = [0.9, 50.0]
-for alg in [Bisection(), Falsi(), Ridder(), Brent()]
+for alg in [Bisection(), Falsi(), Ridder(), Brent(), Itp()]
     global g, p
     g = function (p)
         probN = IntervalNonlinearProblem{false}(f, tspan, p)

From 0517f94f213c963fb5a4354e3a4e9a356f3cc547 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 30 Jun 2023 14:55:58 +0200
Subject: [PATCH 144/375] hyper param change

---
 lib/SimpleNonlinearSolve/src/itp.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index c0d69f595..568bb71b8 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -11,7 +11,7 @@ struct Itp <: AbstractBracketingAlgorithm
     k1::Real
     k2::Real
     n0::Int
-    function Itp(;k1::Real = 0.1, k2::Real = 2.0, n0::Int = 1)
+    function Itp(;k1::Real = 0.3, k2::Real = 2.0, n0::Int = 1)
         if k1 < 0
             error("Hyper-parameter κ₁ should not be negative")
         end
@@ -26,7 +26,7 @@ struct Itp <: AbstractBracketingAlgorithm
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
-                            args...; abstol = 1.0e-10,
+                            args...; abstol = 1.0e-8,
                             maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b

From 28aef82d10b7e4578b6b8ab9423fc1532090bd1a Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 30 Jun 2023 15:15:54 -0400
Subject: [PATCH 145/375] Add Inplace tests

---
 lib/SimpleNonlinearSolve/Project.toml         |  2 +-
 .../src/SimpleNonlinearSolve.jl               |  4 +-
 .../src/batched/dfsane.jl                     |  2 +-
 .../src/batched/lbroyden.jl                   |  1 -
 lib/SimpleNonlinearSolve/test/inplace.jl      | 52 +++++++++++++++++++
 lib/SimpleNonlinearSolve/test/runtests.jl     |  7 +--
 6 files changed, 59 insertions(+), 9 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/inplace.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index f1f472d08..258f009d0 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -21,8 +21,8 @@ LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 
 [extensions]
-SimpleNonlinearSolveNNlibExt = "NNlib"
 SimpleNonlinearSolveADLinearSolveExt = ["AbstractDifferentiation", "LinearSolve"]
+SimpleNonlinearSolveNNlibExt = "NNlib"
 
 [compat]
 AbstractDifferentiation = "0.5"
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 23b332fb3..cd4855612 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -45,7 +45,6 @@ include("batched/utils.jl")
 include("batched/raphson.jl")
 include("batched/dfsane.jl")
 include("batched/broyden.jl")
-include("batched/lbroyden.jl")
 
 import PrecompileTools
 
@@ -79,7 +78,6 @@ end
 # DiffEq styled algorithms
 export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
     Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld
-export BatchedBroyden, SimpleBatchedNewtonRaphson, SimpleBatchedDFSane,
-    BatchedLBroyden
+export BatchedBroyden, SimpleBatchedNewtonRaphson, SimpleBatchedDFSane
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
index 09fc37f8d..a394517a0 100644
--- a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
@@ -1,5 +1,5 @@
 Base.@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition} <:
-              AbstractBatchedNonlinearSolveAlgorithm
+                   AbstractBatchedNonlinearSolveAlgorithm
     σₘᵢₙ::T = 1.0f-10
     σₘₐₓ::T = 1.0f+10
     σ₁::T = 1.0f0
diff --git a/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl b/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
deleted file mode 100644
index 8b1378917..000000000
--- a/lib/SimpleNonlinearSolve/src/batched/lbroyden.jl
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/lib/SimpleNonlinearSolve/test/inplace.jl b/lib/SimpleNonlinearSolve/test/inplace.jl
new file mode 100644
index 000000000..4c43a1da8
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/inplace.jl
@@ -0,0 +1,52 @@
+using SimpleNonlinearSolve, StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
+    NNlib, AbstractDifferentiation, LinearSolve
+
+# Supported Solvers: BatchedBroyden, SimpleBatchedDFSane
+function f!(du::AbstractArray{<:Number, N},
+    u::AbstractArray{<:Number, N},
+    p::AbstractVector) where {N}
+    u_ = reshape(u, :, size(u, N))
+    du .= reshape(sum(abs2, u_; dims = 1) .- reshape(p, 1, :),
+        ntuple(_ -> 1, N - 1)...,
+        size(u, N))
+    return du
+end
+
+function f!(du::AbstractMatrix, u::AbstractMatrix, p::AbstractVector)
+    du .= sum(abs2, u; dims = 1) .- reshape(p, 1, :)
+    return du
+end
+
+function f!(du::AbstractVector, u::AbstractVector, p::AbstractVector)
+    du .= sum(abs2, u) .- p
+    return du
+end
+
+@testset "Solver: $(nameof(typeof(solver)))" for solver in (Broyden(batched = true),
+    SimpleDFSane(batched = true))
+    @testset "T: $T" for T in (Float32, Float64)
+        p = rand(T, 5)
+        @testset "size(u0): $sz" for sz in ((2, 5), (1, 5), (2, 3, 5))
+            u0 = ones(T, sz)
+            prob = NonlinearProblem{true}(f!, u0, p)
+
+            sol = solve(prob, solver)
+
+            @test SciMLBase.successful_retcode(sol.retcode)
+
+            @test sol.resid≈zero(sol.resid) atol=5e-3
+        end
+
+        p = rand(T, 1)
+        @testset "size(u0): $sz" for sz in ((3,), (5,), (10,))
+            u0 = ones(T, sz)
+            prob = NonlinearProblem{true}(f!, u0, p)
+
+            sol = solve(prob, solver)
+
+            @test SciMLBase.successful_retcode(sol.retcode)
+
+            @test sol.resid≈zero(sol.resid) atol=5e-3
+        end
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 94a0086e4..bea57ea0a 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -1,14 +1,15 @@
-using Pkg
 using SafeTestsets
-const LONGER_TESTS = false
 
 const GROUP = get(ENV, "GROUP", "All")
-const is_APPVEYOR = Sys.iswindows() && haskey(ENV, "APPVEYOR")
 
 @time begin
     if GROUP == "All" || GROUP == "Core"
         @time @safetestset "Basic Tests + Some AD" begin
             include("basictests.jl")
         end
+
+        @time @safetestset "Inplace Tests" begin
+            include("inplace.jl")
+        end
     end
 end

From 6c3223915e6a8c0551aea0ebc3667c6c4813fb89 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 30 Jun 2023 15:29:46 -0400
Subject: [PATCH 146/375] Allow v1.6 to work

---
 lib/SimpleNonlinearSolve/src/batched/dfsane.jl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
index a394517a0..a76c768c2 100644
--- a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
@@ -42,7 +42,8 @@ function SciMLBase.__solve(prob::NonlinearProblem,
     α₊, α₋ = similar(u, 1, N), similar(u, 1, N)
     σₙ = fill(T(alg.σ₁), 1, N)
     𝒹 = similar(σₙ, L, N)
-    (; M, nₑₓₚ) = alg
+    M = alg.M
+    nₑₓₚ = alg.nₑₓₚ
 
     xₙ, xₙ₋₁, f₍ₙₒᵣₘ₎ₙ₋₁, f₍ₙₒᵣₘ₎ₙ = copy(u), copy(u), similar(u, 1, N), similar(u, 1, N)
 

From 504ab19c750efa76093d3ef00a41d843777667a7 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 7 Jul 2023 02:26:37 +0200
Subject: [PATCH 147/375] final hyper params, tol and some modifications

---
 lib/SimpleNonlinearSolve/src/itp.jl | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 568bb71b8..5a91df004 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -11,7 +11,7 @@ struct Itp <: AbstractBracketingAlgorithm
     k1::Real
     k2::Real
     n0::Int
-    function Itp(;k1::Real = 0.3, k2::Real = 2.0, n0::Int = 1)
+    function Itp(;k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
         if k1 < 0
             error("Hyper-parameter κ₁ should not be negative")
         end
@@ -26,7 +26,7 @@ struct Itp <: AbstractBracketingAlgorithm
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
-                            args...; abstol = 1.0e-8,
+                            args...; abstol = 1.0e-15,
                             maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b
@@ -46,7 +46,6 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
     k2 = alg.k2
     n0 = alg.n0
     n_h = ceil(log2((right - left) / (2 * ϵ)))
-    n_max = n_h + n0
     mid = (left + right) / 2
     x_f = (fr * left - fl * right) / (fr - fl)
     xt = left
@@ -54,11 +53,12 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
     r = zero(left) #minmax radius
     δ = zero(left) # truncation error
     σ = 1.0
+    ϵ_s = ϵ * 2^(n_h + n0)
     i = 0 #iteration
     while i <= maxiters
         #mid = (left + right) / 2
-        r = ϵ * 2 ^ (n_max - i) - ((right - left) / 2)
-        δ = k1 * (right - left) ^ k2
+        r = ϵ_s - ((right - left) / 2)
+        δ = k1 * ((right - left) ^ k2)
 
         ## Interpolation step ##
         x_f =  (fr * left - fl * right) / (fr - fl)
@@ -92,6 +92,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
         end
         i += 1
         mid = (left + right) / 2
+        ϵ_s /= 2
 
         if (right - left < 2 * ϵ)
             return SciMLBase.build_solution(prob, alg, mid, f(mid);

From bd3d622f519c95266c026aebcd7d2e35e0e9ffa4 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 7 Jul 2023 02:26:51 +0200
Subject: [PATCH 148/375] error check tests

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 34ef7096f..151536005 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -357,6 +357,18 @@ probB = IntervalNonlinearProblem(f, tspan)
 sol = solve(probB, Alefeld())
 @test sol.u ≈ sqrt(2.0)
 
+# Itp
+sol = solve(probB, Itp())
+@test sol.u ≈ sqrt(2.0)
+tspan = (sqrt(2.0), 10.0)
+probB = IntervalNonlinearProblem(f, tspan)
+sol = solve(probB, Itp())
+@test sol.u ≈ sqrt(2.0)
+tspan = (0.0, sqrt(2.0))
+probB = IntervalNonlinearProblem(f, tspan)
+sol = solve(probB, Itp())
+@test sol.u ≈ sqrt(2.0)
+
 # Garuntee Tests for Bisection
 f = function (u, p)
     if u < 2.0

From 8356a887777ba0be091492d2b4d86fee08e8b309 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 7 Jul 2023 02:35:09 +0200
Subject: [PATCH 149/375] format

---
 lib/SimpleNonlinearSolve/src/itp.jl | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 5a91df004..9968cbab1 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -11,7 +11,7 @@ struct Itp <: AbstractBracketingAlgorithm
     k1::Real
     k2::Real
     n0::Int
-    function Itp(;k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
+    function Itp(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
         if k1 < 0
             error("Hyper-parameter κ₁ should not be negative")
         end
@@ -26,8 +26,8 @@ struct Itp <: AbstractBracketingAlgorithm
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
-                            args...; abstol = 1.0e-15,
-                            maxiters = 1000, kwargs...)
+    args...; abstol = 1.0e-15,
+    maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b
     fl, fr = f(left), f(right)
@@ -58,10 +58,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
     while i <= maxiters
         #mid = (left + right) / 2
         r = ϵ_s - ((right - left) / 2)
-        δ = k1 * ((right - left) ^ k2)
+        δ = k1 * ((right - left)^k2)
 
         ## Interpolation step ##
-        x_f =  (fr * left - fl * right) / (fr - fl)
+        x_f = (fr * left - fl * right) / (fr - fl)
 
         ## Truncation step ##
         σ = sign(mid - x_f)
@@ -96,11 +96,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
 
         if (right - left < 2 * ϵ)
             return SciMLBase.build_solution(prob, alg, mid, f(mid);
-            retcode = ReturnCode.Success, left = left,
-            right = right)
+                retcode = ReturnCode.Success, left = left,
+                right = right)
         end
     end
     return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
         left = left, right = right)
-                               
-end
\ No newline at end of file
+end

From 4140c20e2cc4872e0d41025279e9a7eb04145bb7 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 10 Jul 2023 12:47:02 -0400
Subject: [PATCH 150/375] Update batched Raphson to use same parameters as
 unbatched

---
 lib/SimpleNonlinearSolve/Project.toml         |  7 --
 .../SimpleNonlinearSolveADLinearSolveExt.jl   | 90 -------------------
 .../src/SimpleNonlinearSolve.jl               |  3 +-
 .../src/batched/dfsane.jl                     |  4 +-
 .../src/batched/raphson.jl                    | 77 +++++++++++++++-
 lib/SimpleNonlinearSolve/src/dfsane.jl        |  2 +-
 lib/SimpleNonlinearSolve/src/raphson.jl       | 17 ++--
 lib/SimpleNonlinearSolve/test/Project.toml    |  2 -
 lib/SimpleNonlinearSolve/test/basictests.jl   | 12 ++-
 lib/SimpleNonlinearSolve/test/inplace.jl      |  4 +-
 10 files changed, 98 insertions(+), 120 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 258f009d0..89848eb78 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -16,21 +16,16 @@ SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [weakdeps]
-AbstractDifferentiation = "c29ec348-61ec-40c8-8164-b8c60e9d9f3d"
-LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 
 [extensions]
-SimpleNonlinearSolveADLinearSolveExt = ["AbstractDifferentiation", "LinearSolve"]
 SimpleNonlinearSolveNNlibExt = "NNlib"
 
 [compat]
-AbstractDifferentiation = "0.5"
 ArrayInterface = "6, 7"
 DiffEqBase = "6.126"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
-LinearSolve = "2"
 NNlib = "0.8, 0.9"
 PackageExtensionCompat = "1"
 PrecompileTools = "1"
@@ -40,6 +35,4 @@ StaticArraysCore = "1.4"
 julia = "1.6"
 
 [extras]
-AbstractDifferentiation = "c29ec348-61ec-40c8-8164-b8c60e9d9f3d"
-LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
deleted file mode 100644
index d0a97eb86..000000000
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveADLinearSolveExt.jl
+++ /dev/null
@@ -1,90 +0,0 @@
-module SimpleNonlinearSolveADLinearSolveExt
-
-using AbstractDifferentiation,
-    ArrayInterface, DiffEqBase, LinearAlgebra, LinearSolve,
-    SimpleNonlinearSolve, SciMLBase
-import SimpleNonlinearSolve: _construct_batched_problem_structure,
-    _get_storage, _result_from_storage, _get_tolerance, @maybeinplace
-
-const AD = AbstractDifferentiation
-
-function __init__()
-    SimpleNonlinearSolve.ADLinearSolveExtLoaded[] = true
-    return
-end
-
-function SimpleNonlinearSolve.SimpleBatchedNewtonRaphson(; chunk_size = Val{0}(),
-    autodiff = Val{true}(),
-    diff_type = Val{:forward},
-    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-        abstol = nothing,
-        reltol = nothing))
-    # TODO: Use `diff_type`. FiniteDiff.jl is currently not available in AD.jl
-    chunksize = SciMLBase._unwrap_val(chunk_size) == 0 ? nothing : chunk_size
-    ad = SciMLBase._unwrap_val(autodiff) ?
-         AD.ForwardDiffBackend(; chunksize) :
-         AD.FiniteDifferencesBackend()
-    return SimpleBatchedNewtonRaphson{typeof(ad), Nothing, typeof(termination_condition)}(ad,
-        nothing,
-        termination_condition)
-end
-
-function SciMLBase.__solve(prob::NonlinearProblem,
-    alg::SimpleBatchedNewtonRaphson;
-    abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000,
-    kwargs...)
-    iip = isinplace(prob)
-    @assert !iip "SimpleBatchedNewtonRaphson currently only supports out-of-place nonlinear problems."
-    u, f, reconstruct = _construct_batched_problem_structure(prob)
-
-    tc = alg.termination_condition
-    mode = DiffEqBase.get_termination_mode(tc)
-
-    storage = _get_storage(mode, u)
-
-    xₙ, xₙ₋₁, δx = copy(u), copy(u), copy(u)
-    T = eltype(u)
-
-    atol = _get_tolerance(abstol, tc.abstol, T)
-    rtol = _get_tolerance(reltol, tc.reltol, T)
-    termination_condition = tc(storage)
-
-    for i in 1:maxiters
-        fₙ, (𝓙,) = AD.value_and_jacobian(alg.autodiff, f, xₙ)
-
-        iszero(fₙ) && return DiffEqBase.build_solution(prob,
-            alg,
-            reconstruct(xₙ),
-            reconstruct(fₙ);
-            retcode = ReturnCode.Success)
-
-        solve(LinearProblem(𝓙, vec(fₙ); u0 = vec(δx)), alg.linsolve; kwargs...)
-        xₙ .-= δx
-
-        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
-            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
-            return DiffEqBase.build_solution(prob,
-                alg,
-                reconstruct(xₙ),
-                reconstruct(fₙ);
-                retcode)
-        end
-
-        xₙ₋₁ .= xₙ
-    end
-
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        xₙ = storage.u
-        fₙ = f(xₙ)
-    end
-
-    return DiffEqBase.build_solution(prob,
-        alg,
-        reconstruct(xₙ),
-        reconstruct(fₙ);
-        retcode = ReturnCode.MaxIters)
-end
-
-end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index cd4855612..bc57d12cb 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -15,7 +15,6 @@ function __init__()
     @require_extensions
 end
 
-const ADLinearSolveExtLoaded = Ref{Bool}(false)
 const NNlibExtLoaded = Ref{Bool}(false)
 
 abstract type AbstractSimpleNonlinearSolveAlgorithm <: SciMLBase.AbstractNonlinearAlgorithm end
@@ -78,6 +77,6 @@ end
 # DiffEq styled algorithms
 export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
     Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld
-export BatchedBroyden, SimpleBatchedNewtonRaphson, SimpleBatchedDFSane
+export BatchedBroyden, BatchedSimpleNewtonRaphson, BatchedSimpleDFSane
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
index a76c768c2..60bb6ae12 100644
--- a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
@@ -1,4 +1,4 @@
-Base.@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition} <:
+Base.@kwdef struct BatchedSimpleDFSane{T, F, TC <: NLSolveTerminationCondition} <:
                    AbstractBatchedNonlinearSolveAlgorithm
     σₘᵢₙ::T = 1.0f-10
     σₘₐₓ::T = 1.0f+10
@@ -16,7 +16,7 @@ Base.@kwdef struct SimpleBatchedDFSane{T, F, TC <: NLSolveTerminationCondition}
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-    alg::SimpleBatchedDFSane,
+    alg::BatchedSimpleDFSane,
     args...;
     abstol = nothing,
     reltol = nothing,
diff --git a/lib/SimpleNonlinearSolve/src/batched/raphson.jl b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
index 63c03ec89..323c07e73 100644
--- a/lib/SimpleNonlinearSolve/src/batched/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
@@ -1,8 +1,77 @@
-struct SimpleBatchedNewtonRaphson{AD, LS, TC <: NLSolveTerminationCondition} <:
+struct BatchedSimpleNewtonRaphson{CS, AD, FDT, TC <: NLSolveTerminationCondition} <:
        AbstractBatchedNonlinearSolveAlgorithm
-    autodiff::AD
-    linsolve::LS
     termination_condition::TC
 end
 
-# Implementation of solve using Package Extensions
+alg_autodiff(alg::BatchedSimpleNewtonRaphson{CS, AD, FDT}) where {CS, AD, FDT} = AD
+diff_type(alg::BatchedSimpleNewtonRaphson{CS, AD, FDT}) where {CS, AD, FDT} = FDT
+
+function BatchedSimpleNewtonRaphson(; chunk_size = Val{0}(),
+    autodiff = Val{true}(),
+    diff_type = Val{:forward},
+    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+        abstol = nothing,
+        reltol = nothing))
+    return BatchedSimpleNewtonRaphson{SciMLBase._unwrap_val(chunk_size),
+        SciMLBase._unwrap_val(autodiff),
+        SciMLBase._unwrap_val(diff_type), typeof(termination_condition)}(termination_condition)
+end
+
+function SciMLBase.__solve(prob::NonlinearProblem, alg::BatchedSimpleNewtonRaphson;
+    abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
+    iip = SciMLBase.isinplace(prob)
+    @assert !iip "BatchedSimpleNewtonRaphson currently only supports out-of-place nonlinear problems."
+    u, f, reconstruct = _construct_batched_problem_structure(prob)
+
+    tc = alg.termination_condition
+    mode = DiffEqBase.get_termination_mode(tc)
+
+    storage = _get_storage(mode, u)
+
+    xₙ, xₙ₋₁ = copy(u), copy(u)
+    T = eltype(u)
+
+    atol = _get_tolerance(abstol, tc.abstol, T)
+    rtol = _get_tolerance(reltol, tc.reltol, T)
+    termination_condition = tc(storage)
+
+    for i in 1:maxiters
+        if alg_autodiff(alg)
+            fₙ, 𝓙 = value_derivative(f, xₙ)
+        else
+            fₙ = f(xₙ)
+            𝓙 = FiniteDiff.finite_difference_jacobian(f, xₙ, diff_type(alg), eltype(xₙ), fₙ)
+        end
+
+        iszero(fₙ) && return DiffEqBase.build_solution(prob,
+            alg,
+            reconstruct(xₙ),
+            reconstruct(fₙ);
+            retcode = ReturnCode.Success)
+
+        δx = reshape(𝓙 \ vec(fₙ), size(xₙ))
+        xₙ .-= δx
+
+        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
+            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
+            return DiffEqBase.build_solution(prob,
+                alg,
+                reconstruct(xₙ),
+                reconstruct(fₙ);
+                retcode)
+        end
+
+        xₙ₋₁ .= xₙ
+    end
+
+    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
+        xₙ = storage.u
+        fₙ = f(xₙ)
+    end
+
+    return DiffEqBase.build_solution(prob,
+        alg,
+        reconstruct(xₙ),
+        reconstruct(fₙ);
+        retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index d4bc77709..2e52cde4f 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -73,7 +73,7 @@ function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real =
     batched::Bool = false,
     max_inner_iterations = 1000)
     if batched
-        return SimpleBatchedDFSane(; σₘᵢₙ = σ_min,
+        return BatchedSimpleDFSane(; σₘᵢₙ = σ_min,
             σₘₐₓ = σ_max,
             σ₁ = σ_1,
             M,
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 2621c587e..48b8f7591 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -2,7 +2,8 @@
     SimpleNewtonRaphson(; batched = false,
         chunk_size = Val{0}(),
         autodiff = Val{true}(),
-        diff_type = Val{:forward})
+        diff_type = Val{:forward},
+        termination_condition = missing)
 
 A low-overhead implementation of Newton-Raphson. This method is non-allocating on scalar
 and static array problems.
@@ -27,11 +28,8 @@ and static array problems.
 - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
   `Val{:forward}` for forward finite differences. For more details on the choices, see the
   [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
-
-!!! note
-
-    To use the `batched` version, remember to load `AbstractDifferentiation` and
-    `LinearSolve`.
+- `termination_condition`: control the termination of the algorithm. (Only works for batched
+  problems)
 """
 struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT} end
 
@@ -44,16 +42,19 @@ function SimpleNewtonRaphson(; batched = false,
         throw(ArgumentError("`termination_condition` is currently only supported for batched problems"))
     end
     if batched
-        @assert ADLinearSolveExtLoaded[] "Please install and load `LinearSolve.jl` and `AbstractDifferentiation.jl` to use batched Newton-Raphson."
+        # @assert ADLinearSolveFDExtLoaded[] "Please install and load `LinearSolve.jl`, `FiniteDifferences.jl` and `AbstractDifferentiation.jl` to use batched Newton-Raphson."
         termination_condition = ismissing(termination_condition) ?
                                 NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
             abstol = nothing,
             reltol = nothing) :
                                 termination_condition
-        return SimpleBatchedNewtonRaphson(; chunk_size,
+        return BatchedSimpleNewtonRaphson(; chunk_size,
             autodiff,
             diff_type,
             termination_condition)
+        return SimpleNewtonRaphson{SciMLBase._unwrap_val(chunk_size),
+            SciMLBase._unwrap_val(autodiff),
+            SciMLBase._unwrap_val(diff_type)}()
     end
     return SimpleNewtonRaphson{SciMLBase._unwrap_val(chunk_size),
         SciMLBase._unwrap_val(autodiff),
diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
index 280bf2a39..469f302ee 100644
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -1,10 +1,8 @@
 [deps]
-AbstractDifferentiation = "c29ec348-61ec-40c8-8164-b8c60e9d9f3d"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 4fe316b34..4b47c6aaa 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,5 +1,5 @@
 using SimpleNonlinearSolve, StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
-    NNlib, AbstractDifferentiation, LinearSolve
+    NNlib
 
 const BATCHED_BROYDEN_SOLVERS = []
 const BROYDEN_SOLVERS = []
@@ -7,6 +7,7 @@ const BATCHED_LBROYDEN_SOLVERS = []
 const LBROYDEN_SOLVERS = []
 const BATCHED_DFSANE_SOLVERS = []
 const DFSANE_SOLVERS = []
+const BATCHED_RAPHSON_SOLVERS = []
 
 for mode in instances(NLSolveTerminationMode.T)
     if mode ∈
@@ -23,6 +24,12 @@ for mode in instances(NLSolveTerminationMode.T)
     push!(BATCHED_LBROYDEN_SOLVERS, LBroyden(; batched = true, termination_condition))
     push!(DFSANE_SOLVERS, SimpleDFSane(; batched = false, termination_condition))
     push!(BATCHED_DFSANE_SOLVERS, SimpleDFSane(; batched = true, termination_condition))
+    push!(BATCHED_RAPHSON_SOLVERS,
+        SimpleNewtonRaphson(; batched = true,
+            termination_condition))
+    push!(BATCHED_RAPHSON_SOLVERS,
+        SimpleNewtonRaphson(; batched = true, autodiff = false,
+            termination_condition))
 end
 
 # SimpleNewtonRaphson
@@ -483,7 +490,8 @@ sol = solve(probN, Broyden(batched = true))
 
 @testset "Batched Solver: $(nameof(typeof(alg)))" for alg in (BATCHED_BROYDEN_SOLVERS...,
     BATCHED_LBROYDEN_SOLVERS...,
-    BATCHED_DFSANE_SOLVERS...)
+    BATCHED_DFSANE_SOLVERS...,
+    BATCHED_RAPHSON_SOLVERS...)
     sol = solve(probN, alg; abstol = 1e-3, reltol = 1e-3)
 
     @test sol.retcode == ReturnCode.Success
diff --git a/lib/SimpleNonlinearSolve/test/inplace.jl b/lib/SimpleNonlinearSolve/test/inplace.jl
index 4c43a1da8..0f2d74719 100644
--- a/lib/SimpleNonlinearSolve/test/inplace.jl
+++ b/lib/SimpleNonlinearSolve/test/inplace.jl
@@ -1,7 +1,7 @@
 using SimpleNonlinearSolve, StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
-    NNlib, AbstractDifferentiation, LinearSolve
+    NNlib
 
-# Supported Solvers: BatchedBroyden, SimpleBatchedDFSane
+# Supported Solvers: BatchedBroyden, BatchedSimpleDFSane
 function f!(du::AbstractArray{<:Number, N},
     u::AbstractArray{<:Number, N},
     p::AbstractVector) where {N}

From 43a87a722da006250f2edcc0136b8f134fda9991 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 10 Jul 2023 12:49:53 -0400
Subject: [PATCH 151/375] formatting

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 3 ++-
 lib/SimpleNonlinearSolve/test/inplace.jl    | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 4b47c6aaa..d5f85cdae 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,4 +1,5 @@
-using SimpleNonlinearSolve, StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
+using SimpleNonlinearSolve,
+    StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
     NNlib
 
 const BATCHED_BROYDEN_SOLVERS = []
diff --git a/lib/SimpleNonlinearSolve/test/inplace.jl b/lib/SimpleNonlinearSolve/test/inplace.jl
index 0f2d74719..886e8206a 100644
--- a/lib/SimpleNonlinearSolve/test/inplace.jl
+++ b/lib/SimpleNonlinearSolve/test/inplace.jl
@@ -1,4 +1,5 @@
-using SimpleNonlinearSolve, StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
+using SimpleNonlinearSolve,
+    StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
     NNlib
 
 # Supported Solvers: BatchedBroyden, BatchedSimpleDFSane

From f4431a1cf63e0aa3f6c8acc3e3905026c855df36 Mon Sep 17 00:00:00 2001
From: Hossein Pourbozorg <prbzrg@gmail.com>
Date: Mon, 17 Jul 2023 10:58:59 +0330
Subject: [PATCH 152/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index cf5a675af..28d6e7537 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -26,7 +26,7 @@ ArrayInterface = "6, 7"
 DiffEqBase = "6.123.0"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
-NNlib = "0.8"
+NNlib = "0.8, 0.9"
 Reexport = "0.2, 1"
 Requires = "1"
 SciMLBase = "1.73"

From 5cc8b123889524c90da2066ea6203638a442ee2f Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 18 Jul 2023 11:33:30 -0400
Subject: [PATCH 153/375] Update itp.jl

---
 lib/SimpleNonlinearSolve/src/itp.jl | 42 +++++++++++++++++++++++++----
 1 file changed, 37 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 9968cbab1..0a38f312d 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -1,15 +1,47 @@
 """
 ```julia
-Itp(; k1 = Val{1}(), k2 = Val{2}(), n0 = Val{1}())
+Itp(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
 ```
+
 ITP (Interpolate Truncate & Project)
 
+Use the [ITP method](https://en.wikipedia.org/wiki/ITP_method) to find
+a root of a bracketed function, with a convergence rate between 1 and 1.62.
 
-"""
+This method was introduced in the paper "An Enhancement of the Bisection Method
+Average Performance Preserving Minmax Optimality"
+(https://doi.org/10.1145/3423597) by I. F. D. Oliveira and R. H. C. Takahashi.
+
+# Tuning Parameters
+
+The following keyword parameters are accepted.
+
+- `n₀::Int = 1`, the 'slack'. Must not be negative.\n
+  When n₀ = 0 the worst-case is identical to that of bisection,
+  but increacing n₀ provides greater oppotunity for superlinearity.
+- `κ₁::Float64 = 0.1`. Must not be negative.\n
+  The recomended value is `0.2/(x₂ - x₁)`.
+  Lower values produce tighter asymptotic behaviour, while higher values
+  improve the steady-state behaviour when truncation is not helpful.
+- `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62).\n
+  Higher values allow for a greater convergence rate,
+  but also make the method more succeptable to worst-case performance.
+  In practice, κ=1,2 seems to work well due to the computational simplicity,
+  as κ₂ is used as an exponent in the method.
 
-struct Itp <: AbstractBracketingAlgorithm
-    k1::Real
-    k2::Real
+### Worst Case Performance
+
+n½ + `n₀` iterations, where n½ is the number of iterations using bisection
+(n½ = ⌈log2(Δx)/2`tol`⌉).
+
+### Asymptotic Performance
+
+If `f` is twice differentiable and the root is simple,
+then with `n₀` > 0 the convergence rate is √`κ₂`.
+"""
+struct Itp <: AbstractBracketingAlgorithm{T}
+    k1::T
+    k2::T
     n0::Int
     function Itp(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
         if k1 < 0

From 3d52cd68699876c1328e194b1aa5c964ea943a6f Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 18 Jul 2023 12:10:10 -0400
Subject: [PATCH 154/375] Update src/itp.jl

---
 lib/SimpleNonlinearSolve/src/itp.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 0a38f312d..b9cc28cda 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -39,7 +39,7 @@ n½ + `n₀` iterations, where n½ is the number of iterations using bisection
 If `f` is twice differentiable and the root is simple,
 then with `n₀` > 0 the convergence rate is √`κ₂`.
 """
-struct Itp <: AbstractBracketingAlgorithm{T}
+struct Itp{T} <: AbstractBracketingAlgorithm
     k1::T
     k2::T
     n0::Int

From c01cf7b6d802fea7d6444c0d3d55c940ef28975c Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 18 Jul 2023 12:22:21 -0400
Subject: [PATCH 155/375] Update src/itp.jl

---
 lib/SimpleNonlinearSolve/src/itp.jl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index b9cc28cda..8e60ff1ad 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -53,7 +53,8 @@ struct Itp{T} <: AbstractBracketingAlgorithm
         if k2 < 1 || k2 > (1.5 + sqrt(5) / 2)
             ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where ϕ ≈ 1.618... is the golden ratio")
         end
-        return new(k1, k2, n0)
+        T = promote_type(eltype(k1), eltype(k2))
+        return new{T}(k1, k2, n0)
     end
 end
 

From 091a9239cda887750ac1d8e6ce3070a17f7a8b49 Mon Sep 17 00:00:00 2001
From: oscarddssmith <oscar.smith@juliacomputing.com>
Date: Tue, 18 Jul 2023 13:50:48 -0400
Subject: [PATCH 156/375] capitalize ITP

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl |  4 ++--
 lib/SimpleNonlinearSolve/src/itp.jl                  |  8 ++++----
 lib/SimpleNonlinearSolve/test/basictests.jl          | 12 ++++++------
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index bebbc6143..5c500d500 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -69,7 +69,7 @@ PrecompileTools.@compile_workload begin
         prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
             T.((0.0, 2.0)),
             T(2))
-        for alg in (Bisection, Falsi, Ridder, Brent, Alefeld, Itp)
+        for alg in (Bisection, Falsi, Ridder, Brent, Alefeld, ITP)
             solve(prob_brack, alg(), abstol = T(1e-2))
         end
     end
@@ -77,7 +77,7 @@ end
 
 # DiffEq styled algorithms
 export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
-    Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld, Itp
+    Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld, ITP
 export BatchedBroyden, BatchedSimpleNewtonRaphson, BatchedSimpleDFSane
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 8e60ff1ad..4483fc0ab 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -1,6 +1,6 @@
 """
 ```julia
-Itp(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
+ITP(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
 ```
 
 ITP (Interpolate Truncate & Project)
@@ -39,11 +39,11 @@ n½ + `n₀` iterations, where n½ is the number of iterations using bisection
 If `f` is twice differentiable and the root is simple,
 then with `n₀` > 0 the convergence rate is √`κ₂`.
 """
-struct Itp{T} <: AbstractBracketingAlgorithm
+struct ITP{T} <: AbstractBracketingAlgorithm
     k1::T
     k2::T
     n0::Int
-    function Itp(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
+    function ITP(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
         if k1 < 0
             error("Hyper-parameter κ₁ should not be negative")
         end
@@ -58,7 +58,7 @@ struct Itp{T} <: AbstractBracketingAlgorithm
     end
 end
 
-function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Itp,
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
     args...; abstol = 1.0e-15,
     maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 72545af06..38dce60da 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -222,7 +222,7 @@ end
 # ITP
 g = function (p)
     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-    sol = solve(probN, Itp())
+    sol = solve(probN, ITP())
     return sol.u
 end
 
@@ -258,7 +258,7 @@ end
 f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
 t = (p) -> [sqrt(p[2] / p[1])]
 p = [0.9, 50.0]
-for alg in [Bisection(), Falsi(), Ridder(), Brent(), Itp()]
+for alg in [Bisection(), Falsi(), Ridder(), Brent(), ITP()]
     global g, p
     g = function (p)
         probN = IntervalNonlinearProblem{false}(f, tspan, p)
@@ -361,16 +361,16 @@ probB = IntervalNonlinearProblem(f, tspan)
 sol = solve(probB, Alefeld())
 @test sol.u ≈ sqrt(2.0)
 
-# Itp
-sol = solve(probB, Itp())
+# ITP
+sol = solve(probB, ITP())
 @test sol.u ≈ sqrt(2.0)
 tspan = (sqrt(2.0), 10.0)
 probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, Itp())
+sol = solve(probB, ITP())
 @test sol.u ≈ sqrt(2.0)
 tspan = (0.0, sqrt(2.0))
 probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, Itp())
+sol = solve(probB, ITP())
 @test sol.u ≈ sqrt(2.0)
 
 # Garuntee Tests for Bisection

From feb39a24d050f28df0fb87d86453d46ab4fd719c Mon Sep 17 00:00:00 2001
From: oscarddssmith <oscar.smith@juliacomputing.com>
Date: Tue, 18 Jul 2023 13:51:49 -0400
Subject: [PATCH 157/375] bump version

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 89848eb78..e4c556f47 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.17"
+version = "0.1.18"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

From a6757d6444548f535014c95f391bfb64bc6b3f6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez?= <daniel.glez.arribas@gmail.com>
Date: Fri, 21 Jul 2023 14:22:03 +0200
Subject: [PATCH 158/375] Fix ITP for problems with flipped sign

---
 lib/SimpleNonlinearSolve/src/itp.jl         |  5 +++--
 lib/SimpleNonlinearSolve/test/basictests.jl | 16 ++++++++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 4483fc0ab..648208b80 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -113,10 +113,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
 
         ## Update ##
         yp = f(xp)
-        if yp > 0
+        yps = yp * sign(fr)
+        if yps > 0
             right = xp
             fr = yp
-        elseif yp < 0
+        elseif yps < 0
             left = xp
             fl = yp
         else
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 38dce60da..ee3170ffb 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -539,3 +539,19 @@ for alg in (SimpleNewtonRaphson(), SimpleTrustRegion())
     sol = solve(probN, alg)
     @test abs.(sol.u) ≈ sqrt.(p)
 end
+
+# Flipped signs test
+f1(u, p) = u * u - p 
+f2(u, p) = p - u * u
+
+for Alg in (Alefeld, Bisection, Falsi, Brent, ITP, Ridder)
+    alg = Alg()
+    for p ∈ 1:4
+        inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
+        inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
+        sol = solve(inp1, alg)
+        @test abs.(sol.u) ≈ sqrt.(p)
+        sol = solve(inp2, alg)
+        @test abs.(sol.u) ≈ sqrt.(p)
+    end
+end
\ No newline at end of file

From 245483e4104840f66273bf20255b3e1b09ecffb6 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 21 Jul 2023 09:03:11 -0400
Subject: [PATCH 159/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index e4c556f47..ce68fc7b5 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.18"
+version = "0.1.19"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

From 56c8669745b188e768986bc874de0fd3dbb78731 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 21 Jul 2023 09:03:57 -0400
Subject: [PATCH 160/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index ce68fc7b5..e4c556f47 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.19"
+version = "0.1.18"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

From 15f05934655ef03ad02895d54bcc563d99704f63 Mon Sep 17 00:00:00 2001
From: ChrisRackauckas <ChrisRackauckas@users.noreply.github.com>
Date: Sat, 22 Jul 2023 00:20:08 +0000
Subject: [PATCH 161/375] Format .jl files

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index ee3170ffb..9dc681f3a 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -541,12 +541,12 @@ for alg in (SimpleNewtonRaphson(), SimpleTrustRegion())
 end
 
 # Flipped signs test
-f1(u, p) = u * u - p 
+f1(u, p) = u * u - p
 f2(u, p) = p - u * u
 
 for Alg in (Alefeld, Bisection, Falsi, Brent, ITP, Ridder)
     alg = Alg()
-    for p ∈ 1:4
+    for p in 1:4
         inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
         inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
         sol = solve(inp1, alg)
@@ -554,4 +554,4 @@ for Alg in (Alefeld, Bisection, Falsi, Brent, ITP, Ridder)
         sol = solve(inp2, alg)
         @test abs.(sol.u) ≈ sqrt.(p)
     end
-end
\ No newline at end of file
+end

From 319e95b6ec8313554f51433fe5163da03f3292bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez?= <daniel.glez.arribas@gmail.com>
Date: Sun, 23 Jul 2023 14:32:29 +0200
Subject: [PATCH 162/375] ITP improvements

---
 lib/SimpleNonlinearSolve/src/itp.jl         | 22 ++++++++++++---------
 lib/SimpleNonlinearSolve/test/basictests.jl | 15 +++++++-------
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 648208b80..b762eaea9 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -78,7 +78,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
     k1 = alg.k1
     k2 = alg.k2
     n0 = alg.n0
-    n_h = ceil(log2((right - left) / (2 * ϵ)))
+    n_h = ceil(log2(abs(right - left) / (2 * ϵ)))
     mid = (left + right) / 2
     x_f = (fr * left - fl * right) / (fr - fl)
     xt = left
@@ -89,9 +89,9 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
     ϵ_s = ϵ * 2^(n_h + n0)
     i = 0 #iteration
     while i <= maxiters
-        #mid = (left + right) / 2
-        r = ϵ_s - ((right - left) / 2)
-        δ = k1 * ((right - left)^k2)
+        span = abs(right - left)
+        r = ϵ_s - (span / 2)
+        δ = k1 * (span^k2)
 
         ## Interpolation step ##
         x_f = (fr * left - fl * right) / (fr - fl)
@@ -112,6 +112,9 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
         end
 
         ## Update ##
+        tmin, tmax = minmax(left, right)
+        xp >= tmax && (xp = prevfloat(tmax))
+        xp <= tmin && (xp = nextfloat(tmin))
         yp = f(xp)
         yps = yp * sign(fr)
         if yps > 0
@@ -121,16 +124,17 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
             left = xp
             fl = yp
         else
-            left = xp
-            right = xp
+            return SciMLBase.build_solution(prob, alg, xp, yps;
+                                            retcode = ReturnCode.Success, left = left,
+                                            right = right)
         end
         i += 1
         mid = (left + right) / 2
         ϵ_s /= 2
 
-        if (right - left < 2 * ϵ)
-            return SciMLBase.build_solution(prob, alg, mid, f(mid);
-                retcode = ReturnCode.Success, left = left,
+        if nextfloat_tdir(left, prob.tspan...) == right
+            return SciMLBase.build_solution(prob, alg, left, fl;
+                retcode = ReturnCode.FloatingPointLimit, left = left,
                 right = right)
         end
     end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index ee3170ffb..678cf3a6a 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -540,18 +540,19 @@ for alg in (SimpleNewtonRaphson(), SimpleTrustRegion())
     @test abs.(sol.u) ≈ sqrt.(p)
 end
 
-# Flipped signs test
+# Flipped signs & reversed tsoan test for bracketing algorithms
 f1(u, p) = u * u - p 
 f2(u, p) = p - u * u
 
-for Alg in (Alefeld, Bisection, Falsi, Brent, ITP, Ridder)
-    alg = Alg()
+for alg in (Alefeld(), Bisection(), Falsi(), Brent(), ITP(), Ridder())
     for p ∈ 1:4
         inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
         inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
-        sol = solve(inp1, alg)
-        @test abs.(sol.u) ≈ sqrt.(p)
-        sol = solve(inp2, alg)
-        @test abs.(sol.u) ≈ sqrt.(p)
+        inp3 = IntervalNonlinearProblem(f1, (2.0, 1.0), p)
+        inp4 = IntervalNonlinearProblem(f2, (2.0, 1.0), p)
+        @test abs.(solve(inp1, alg).u) ≈ sqrt.(p)
+        @test abs.(solve(inp2, alg).u) ≈ sqrt.(p)
+        @test abs.(solve(inp3, alg).u) ≈ sqrt.(p)
+        @test abs.(solve(inp4, alg).u) ≈ sqrt.(p)
     end
 end
\ No newline at end of file

From 8e8ef86e67b9f0ce264fdfe5389cc407970834f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez?= <daniel.glez.arribas@gmail.com>
Date: Sun, 23 Jul 2023 14:39:15 +0200
Subject: [PATCH 163/375] Use safer formula for falsi point in ITP

---
 lib/SimpleNonlinearSolve/src/itp.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index b762eaea9..2d8c0a69f 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -80,7 +80,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
     n0 = alg.n0
     n_h = ceil(log2(abs(right - left) / (2 * ϵ)))
     mid = (left + right) / 2
-    x_f = (fr * left - fl * right) / (fr - fl)
+    x_f = left + (right - left) * (fl/(fl - fr))
     xt = left
     xp = left
     r = zero(left) #minmax radius
@@ -94,7 +94,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
         δ = k1 * (span^k2)
 
         ## Interpolation step ##
-        x_f = (fr * left - fl * right) / (fr - fl)
+        x_f = left + (right - left) * (fl/(fl - fr))
 
         ## Truncation step ##
         σ = sign(mid - x_f)

From 65f24fe17c1e3bb8106fb06c6c6bc5935165c71e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez?= <daniel.glez.arribas@gmail.com>
Date: Sun, 23 Jul 2023 14:45:08 +0200
Subject: [PATCH 164/375] =?UTF-8?q?Refine=20ITP=20return=20solution=20on?=
 =?UTF-8?q?=20hitting=20f(=C2=B7)=20zero?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 lib/SimpleNonlinearSolve/src/itp.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 2d8c0a69f..1f1efbafe 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -125,8 +125,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
             fl = yp
         else
             return SciMLBase.build_solution(prob, alg, xp, yps;
-                                            retcode = ReturnCode.Success, left = left,
-                                            right = right)
+                                            retcode = ReturnCode.Success, left = xp,
+                                            right = xp)
         end
         i += 1
         mid = (left + right) / 2

From fbd4f68614d02689c080a2af173ed863da660a87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez?= <daniel.glez.arribas@gmail.com>
Date: Sun, 23 Jul 2023 14:45:45 +0200
Subject: [PATCH 165/375] typo

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 95f8d58e2..6ab9e56d8 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -540,7 +540,7 @@ for alg in (SimpleNewtonRaphson(), SimpleTrustRegion())
     @test abs.(sol.u) ≈ sqrt.(p)
 end
 
-# Flipped signs & reversed tsoan test for bracketing algorithms
+# Flipped signs & reversed tspan test for bracketing algorithms
 f1(u, p) = u * u - p
 f2(u, p) = p - u * u
 

From 1fdd283990f06aacf31480397013bd70d1664526 Mon Sep 17 00:00:00 2001
From: ChrisRackauckas <ChrisRackauckas@users.noreply.github.com>
Date: Tue, 25 Jul 2023 00:20:59 +0000
Subject: [PATCH 166/375] Format .jl files

---
 lib/SimpleNonlinearSolve/src/itp.jl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 1f1efbafe..fa390aa45 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -80,7 +80,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
     n0 = alg.n0
     n_h = ceil(log2(abs(right - left) / (2 * ϵ)))
     mid = (left + right) / 2
-    x_f = left + (right - left) * (fl/(fl - fr))
+    x_f = left + (right - left) * (fl / (fl - fr))
     xt = left
     xp = left
     r = zero(left) #minmax radius
@@ -94,7 +94,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
         δ = k1 * (span^k2)
 
         ## Interpolation step ##
-        x_f = left + (right - left) * (fl/(fl - fr))
+        x_f = left + (right - left) * (fl / (fl - fr))
 
         ## Truncation step ##
         σ = sign(mid - x_f)
@@ -125,8 +125,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
             fl = yp
         else
             return SciMLBase.build_solution(prob, alg, xp, yps;
-                                            retcode = ReturnCode.Success, left = xp,
-                                            right = xp)
+                retcode = ReturnCode.Success, left = xp,
+                right = xp)
         end
         i += 1
         mid = (left + right) / 2

From dfb620f74b8d97f992d0077f9c0378e135e22799 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 1 Aug 2023 11:43:55 -0400
Subject: [PATCH 167/375] Add support for inplace BatchedSimpleNewtonRaphson

---
 lib/SimpleNonlinearSolve/Project.toml         |  2 +-
 .../src/batched/raphson.jl                    | 27 ++++++++++++++-----
 lib/SimpleNonlinearSolve/src/utils.jl         | 14 ++++++++++
 lib/SimpleNonlinearSolve/test/inplace.jl      | 15 +++++------
 4 files changed, 43 insertions(+), 15 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index e4c556f47..ce68fc7b5 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.18"
+version = "0.1.19"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
diff --git a/lib/SimpleNonlinearSolve/src/batched/raphson.jl b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
index 323c07e73..a141819bc 100644
--- a/lib/SimpleNonlinearSolve/src/batched/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
@@ -20,7 +20,8 @@ end
 function SciMLBase.__solve(prob::NonlinearProblem, alg::BatchedSimpleNewtonRaphson;
     abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
     iip = SciMLBase.isinplace(prob)
-    @assert !iip "BatchedSimpleNewtonRaphson currently only supports out-of-place nonlinear problems."
+    iip &&
+        @assert alg_autodiff(alg) "Inplace BatchedSimpleNewtonRaphson currently only supports autodiff."
     u, f, reconstruct = _construct_batched_problem_structure(prob)
 
     tc = alg.termination_condition
@@ -35,12 +36,26 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::BatchedSimpleNewtonRaphs
     rtol = _get_tolerance(reltol, tc.reltol, T)
     termination_condition = tc(storage)
 
+    if iip
+        𝓙 = similar(xₙ, length(xₙ), length(xₙ))
+        fₙ = similar(xₙ)
+        jac_cfg = ForwardDiff.JacobianConfig(f, fₙ, xₙ)
+    end
+
     for i in 1:maxiters
-        if alg_autodiff(alg)
-            fₙ, 𝓙 = value_derivative(f, xₙ)
+        if iip
+            value_derivative!(𝓙, fₙ, f, xₙ, jac_cfg)
         else
-            fₙ = f(xₙ)
-            𝓙 = FiniteDiff.finite_difference_jacobian(f, xₙ, diff_type(alg), eltype(xₙ), fₙ)
+            if alg_autodiff(alg)
+                fₙ, 𝓙 = value_derivative(f, xₙ)
+            else
+                fₙ = f(xₙ)
+                𝓙 = FiniteDiff.finite_difference_jacobian(f,
+                    xₙ,
+                    diff_type(alg),
+                    eltype(xₙ),
+                    fₙ)
+            end
         end
 
         iszero(fₙ) && return DiffEqBase.build_solution(prob,
@@ -66,7 +81,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::BatchedSimpleNewtonRaphs
 
     if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
         xₙ = storage.u
-        fₙ = f(xₙ)
+        @maybeinplace iip fₙ=f(xₙ)
     end
 
     return DiffEqBase.build_solution(prob,
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 890aa2415..12462a05c 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -30,6 +30,20 @@ function value_derivative(f::F, x::R) where {F, R}
 end
 value_derivative(f::F, x::AbstractArray) where {F} = f(x), ForwardDiff.jacobian(f, x)
 
+"""
+    value_derivative!(J, y, f!, x, cfg = JacobianConfig(f!, y, x))
+
+Inplace version of [`SimpleNonlinearSolve.value_derivative`](@ref).
+"""
+function value_derivative!(J::AbstractMatrix,
+    y::AbstractArray,
+    f!::F,
+    x::AbstractArray,
+    cfg::ForwardDiff.JacobianConfig = ForwardDiff.JacobianConfig(f!, y, x)) where {F}
+    ForwardDiff.jacobian!(J, f!, y, x, cfg)
+    return y, J
+end
+
 value(x) = x
 value(x::Dual) = ForwardDiff.value(x)
 value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
diff --git a/lib/SimpleNonlinearSolve/test/inplace.jl b/lib/SimpleNonlinearSolve/test/inplace.jl
index 886e8206a..d4882105b 100644
--- a/lib/SimpleNonlinearSolve/test/inplace.jl
+++ b/lib/SimpleNonlinearSolve/test/inplace.jl
@@ -2,29 +2,28 @@ using SimpleNonlinearSolve,
     StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
     NNlib
 
-# Supported Solvers: BatchedBroyden, BatchedSimpleDFSane
+# Supported Solvers: BatchedBroyden, BatchedSimpleDFSane, BatchedSimpleNewtonRaphson
 function f!(du::AbstractArray{<:Number, N},
     u::AbstractArray{<:Number, N},
     p::AbstractVector) where {N}
     u_ = reshape(u, :, size(u, N))
-    du .= reshape(sum(abs2, u_; dims = 1) .- reshape(p, 1, :),
-        ntuple(_ -> 1, N - 1)...,
-        size(u, N))
+    du .= reshape(sum(abs2, u_; dims = 1) .- u_ .- reshape(p, 1, :), size(u))
     return du
 end
 
 function f!(du::AbstractMatrix, u::AbstractMatrix, p::AbstractVector)
-    du .= sum(abs2, u; dims = 1) .- reshape(p, 1, :)
+    du .= sum(abs2, u; dims = 1) .- u .- reshape(p, 1, :)
     return du
 end
 
 function f!(du::AbstractVector, u::AbstractVector, p::AbstractVector)
-    du .= sum(abs2, u) .- p
+    du .= sum(abs2, u) .- u .- p
     return du
 end
 
-@testset "Solver: $(nameof(typeof(solver)))" for solver in (Broyden(batched = true),
-    SimpleDFSane(batched = true))
+@testset "Solver: $(nameof(typeof(solver)))" for solver in (Broyden(; batched = true),
+    SimpleDFSane(; batched = true),
+    SimpleNewtonRaphson(; batched = true))
     @testset "T: $T" for T in (Float32, Float64)
         p = rand(T, 5)
         @testset "size(u0): $sz" for sz in ((2, 5), (1, 5), (2, 3, 5))

From 5b0ec527e2c74dd778ba31924556a4905fb18780 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 1 Sep 2023 02:03:41 +0200
Subject: [PATCH 168/375] tolerance fix bisection

---
 lib/SimpleNonlinearSolve/src/bisection.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 673f77067..6e39afda9 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -20,12 +20,12 @@ function Bisection(; exact_left = false, exact_right = false)
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...;
-    maxiters = 1000,
+    maxiters = 1000, abstol = nothing,
     kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-
+    atol = abstol !== nothing ? abstol : eps(1.0)^(3 // 5)
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,

From 12267cd4cb751ac612abe0e1e7c3a19f1fb16865 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 1 Sep 2023 02:28:42 +0200
Subject: [PATCH 169/375] right exact solution

---
 lib/SimpleNonlinearSolve/src/bisection.jl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 6e39afda9..58f31f184 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -31,6 +31,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
             retcode = ReturnCode.ExactSolutionLeft, left = left,
             right = right)
     end
+    if iszero(fr)
+        return SciMLBase.build_solution(prob, alg, right, fr;
+            retcode = ReturnCode.ExactSolutionRight, left = left,
+            right = right)
+    end
 
     i = 1
     if !iszero(fr)

From b44c6438b0494da485e4997af3955e83f8ea388f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Sep 2023 21:25:34 +0000
Subject: [PATCH 170/375] Bump actions/checkout from 3 to 4

Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v3...v4)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 lib/SimpleNonlinearSolve/.github/workflows/CI.yml            | 2 +-
 lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml    | 4 ++--
 lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml   | 2 +-
 lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml      | 2 +-
 lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml | 4 ++--
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index fc96a93a4..1a6859e3a 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -17,7 +17,7 @@ jobs:
           - '1'
           - '1.6'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
index c6648dfb4..4b1032411 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
@@ -25,14 +25,14 @@ jobs:
           - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceIV}
           - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceV}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.julia-version }}
           arch: x64
       - uses: julia-actions/julia-buildpkg@latest
       - name: Clone Downstream
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{ matrix.package.user }}/${{ matrix.package.repo }}
           path: downstream
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
index 521b8c2b2..45bd09c47 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
@@ -21,7 +21,7 @@ jobs:
         with:
           version: ${{ matrix.julia-version }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install JuliaFormatter and format
         # This will use the latest version by default but you can set the version like so:
         #
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml
index b0cb28af5..eea22ebce 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml
@@ -6,7 +6,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install JuliaFormatter and format
         run: |
           julia  -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter"))'
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml b/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
index 4d0004e83..28b9ce2fa 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
@@ -19,12 +19,12 @@ jobs:
     - uses: julia-actions/setup-julia@v1
       with:
         version: '1'
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: julia-actions/julia-buildpkg@v1
     - uses: julia-actions/julia-invalidations@v1
       id: invs_pr
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.repository.default_branch }}
     - uses: julia-actions/julia-buildpkg@v1

From 0d54cc553a886ff12b6ff61e150fad995feda58f Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 5 Sep 2023 00:37:54 +0200
Subject: [PATCH 171/375] tol for Bisection()

---
 lib/SimpleNonlinearSolve/src/bisection.jl | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 58f31f184..1cb0e5d54 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -46,6 +46,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
                     retcode = ReturnCode.FloatingPointLimit,
                     left = left, right = right)
             fm = f(mid)
+            if abs((right - left) / 2) < atol
+                return SciMLBase.build_solution(prob, alg, mid, fm;
+                    retcode = ReturnCode.Success,
+                    left = left, right = right)
+            end
             if iszero(fm)
                 right = mid
                 break
@@ -68,6 +73,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
                 retcode = ReturnCode.FloatingPointLimit,
                 left = left, right = right)
         fm = f(mid)
+        if abs((right - left) / 2) < atol
+            return SciMLBase.build_solution(prob, alg, mid, fm;
+                retcode = ReturnCode.Success,
+                left = left, right = right)
+        end
         if iszero(fm)
             right = mid
             fr = fm

From a76441b54c2f302eaa2ec319c7981b2a61f6e57b Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Wed, 6 Sep 2023 00:09:42 +0200
Subject: [PATCH 172/375] tests for tol

changed default tol to machine epsilon
---
 lib/SimpleNonlinearSolve/src/bisection.jl   | 2 +-
 lib/SimpleNonlinearSolve/test/basictests.jl | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 1cb0e5d54..17836caf4 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -25,7 +25,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : eps(1.0)^(3 // 5)
+    atol = abstol !== nothing ? abstol : eps(1.0)
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 6ab9e56d8..a1e2c5669 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -373,6 +373,15 @@ probB = IntervalNonlinearProblem(f, tspan)
 sol = solve(probB, ITP())
 @test sol.u ≈ sqrt(2.0)
 
+# Tolerance tests for Interval methods
+
+tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
+
+for abstol in tols
+    sol = solve(probB, Bisection())
+    @test abs(sol.u - sqrt(2)) < abstol
+end
+
 # Garuntee Tests for Bisection
 f = function (u, p)
     if u < 2.0

From 1dfb5dc0edf8467601668fa8996fef1e7cd79717 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 7 Sep 2023 21:01:04 +0200
Subject: [PATCH 173/375] tol fix for brent, ridder and falsi

---
 lib/SimpleNonlinearSolve/src/brent.jl  | 13 ++++++++++++-
 lib/SimpleNonlinearSolve/src/falsi.jl  | 17 ++++++++++++++++-
 lib/SimpleNonlinearSolve/src/ridder.jl | 17 ++++++++++++++++-
 3 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index 1cedad134..2618211bd 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -7,12 +7,13 @@ A non-allocating Brent method
 struct Brent <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
-    maxiters = 1000,
+    maxiters = 1000, abstol = nothing,
     kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     fa, fb = f(a), f(b)
     ϵ = eps(convert(typeof(fa), 1.0))
+    atol = abstol !== nothing ? abstol : eps(1.0)
 
     if iszero(fa)
         return SciMLBase.build_solution(prob, alg, a, fa;
@@ -60,6 +61,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
                 cond = false
             end
             fs = f(s)
+            if abs((b - a) / 2) < atol
+                return SciMLBase.build_solution(prob, alg, s, fs;
+                    retcode = ReturnCode.Success,
+                    left = a, right = b)
+            end
             if iszero(fs)
                 if b < a
                     a = b
@@ -99,6 +105,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
                 left = a, right = b)
         end
         fc = f(c)
+        if abs((b - a) / 2) < atol
+            return SciMLBase.build_solution(prob, alg, c, fc;
+                retcode = ReturnCode.Success,
+                left = a, right = b)
+        end
         if iszero(fc)
             b = c
             fb = fc
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index cce11a811..b08a2a1d1 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -4,16 +4,21 @@
 struct Falsi <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
-    maxiters = 1000,
+    maxiters = 1000, abstol = nothing,
     kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
+    atol = abstol !== nothing ? abstol : eps(1.0)
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,
             right = right)
+    elseif iszero(fr)
+        return SciMLBase.build_solution(prob, alg, right, fr;
+            retcode = ReturnCode.ExactSolutionRight, left = left,
+            right = right)
     end
 
     i = 1
@@ -32,6 +37,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
                 break
             end
             fm = f(mid)
+            if abs((right - left) / 2) < atol
+                return SciMLBase.build_solution(prob, alg, mid, fm;
+                    retcode = ReturnCode.Success,
+                    left = left, right = right)
+            end
             if iszero(fm)
                 right = mid
                 break
@@ -54,6 +64,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
                 retcode = ReturnCode.FloatingPointLimit,
                 left = left, right = right)
         fm = f(mid)
+        if abs((right - left) / 2) < atol
+            return SciMLBase.build_solution(prob, alg, mid, fm;
+                retcode = ReturnCode.Success,
+                left = left, right = right)
+        end
         if iszero(fm)
             right = mid
             fr = fm
diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index 62b5a931a..099891ddd 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -7,16 +7,21 @@ A non-allocating ridder method
 struct Ridder <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
-    maxiters = 1000,
+    maxiters = 1000, abstol = nothing,
     kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
+    atol = abstol !== nothing ? abstol : eps(1.0)
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,
             right = right)
+    elseif iszero(fr)
+        return SciMLBase.build_solution(prob, alg, right, fr;
+            retcode = ReturnCode.ExactSolutionRight, left = left,
+            right = right)
     end
 
     xo = oftype(left, Inf)
@@ -37,6 +42,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
             x = mid + (mid - left) * sign(fl - fr) * fm / s
             fx = f(x)
             xo = x
+            if abs((right - left) / 2) < atol
+                return SciMLBase.build_solution(prob, alg, mid, fm;
+                    retcode = ReturnCode.Success,
+                    left = left, right = right)
+            end
             if iszero(fx)
                 right = x
                 fr = fx
@@ -66,6 +76,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
                 retcode = ReturnCode.FloatingPointLimit,
                 left = left, right = right)
         fm = f(mid)
+        if abs((right - left) / 2) < atol
+            return SciMLBase.build_solution(prob, alg, mid, fm;
+                retcode = ReturnCode.Success,
+                left = left, right = right)
+        end
         if iszero(fm)
             right = mid
             fr = fm

From c0d76ac6296046490993e03f933aabd9e4738af4 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 7 Sep 2023 21:01:31 +0200
Subject: [PATCH 174/375] tol tests - both upper and lower bounds

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 26 +++++++++++++++++----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index a1e2c5669..23253240a 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -374,12 +374,28 @@ sol = solve(probB, ITP())
 @test sol.u ≈ sqrt(2.0)
 
 # Tolerance tests for Interval methods
-
+f, tspan = (u, p) -> u .* u .- 2.0, (1.0, 10.0)
+probB = IntervalNonlinearProblem(f, tspan)
 tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
-
-for abstol in tols
-    sol = solve(probB, Bisection())
-    @test abs(sol.u - sqrt(2)) < abstol
+ϵ = eps(1.0) #least possible tol for all methods
+
+for atol in tols
+    sol = solve(probB, Bisection(), abstol = atol)
+    @test abs(sol.u - sqrt(2)) < atol
+    @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
+    sol = solve(probB, Falsi(), abstol = atol)
+    @test abs(sol.u - sqrt(2)) < atol
+    @test abs(sol.u - sqrt(2)) > ϵ
+end
+
+tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution
+for atol in tols
+    sol = solve(probB, Ridder(), abstol = atol)
+    @test abs(sol.u - sqrt(2)) < atol
+    @test abs(sol.u - sqrt(2)) > ϵ
+    sol = solve(probB, Brent(), abstol = atol)
+    @test abs(sol.u - sqrt(2)) < atol
+    @test abs(sol.u - sqrt(2)) > ϵ
 end
 
 # Garuntee Tests for Bisection

From 10bb570b40a631e9e038b40a621f1165b2915fd7 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Thu, 7 Sep 2023 21:03:36 +0200
Subject: [PATCH 175/375] right check for brent

---
 lib/SimpleNonlinearSolve/src/brent.jl | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index 2618211bd..a6706248a 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -19,6 +19,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
         return SciMLBase.build_solution(prob, alg, a, fa;
             retcode = ReturnCode.ExactSolutionLeft, left = a,
             right = b)
+    elseif iszero(fb)
+        return SciMLBase.build_solution(prob, alg, b, fb;
+            retcode = ReturnCode.ExactSolutionRight, left = a,
+            right = b)
     end
     if abs(fa) < abs(fb)
         c = b

From 62e0845bbe44b8a2c11215157cd7e65d7da8ce58 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 9 Sep 2023 05:09:32 -0400
Subject: [PATCH 176/375] Update src/ridder.jl

---
 lib/SimpleNonlinearSolve/src/ridder.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index 099891ddd..190a37153 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -12,7 +12,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : eps(1.0)
+    atol = abstol !== nothing ? abstol : minimum(eps(left), eps(right))
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;

From 703d984c6de1b4830e697e0f9d223252d117e18f Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 9 Sep 2023 05:09:38 -0400
Subject: [PATCH 177/375] Update src/falsi.jl

---
 lib/SimpleNonlinearSolve/src/falsi.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index b08a2a1d1..549a52c13 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -9,7 +9,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : eps(1.0)
+    atol = abstol !== nothing ? abstol : minimum(eps(left), eps(right))
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;

From c378fad4c50d686951c45712af7c6a18a7cc7681 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 9 Sep 2023 05:09:47 -0400
Subject: [PATCH 178/375] Update src/brent.jl

---
 lib/SimpleNonlinearSolve/src/brent.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index a6706248a..1ddc0f348 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -13,7 +13,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
     a, b = prob.tspan
     fa, fb = f(a), f(b)
     ϵ = eps(convert(typeof(fa), 1.0))
-    atol = abstol !== nothing ? abstol : eps(1.0)
+    atol = abstol !== nothing ? abstol : minimum(eps(left), eps(right))
 
     if iszero(fa)
         return SciMLBase.build_solution(prob, alg, a, fa;

From 11b7d02b0c9d4f890cce360c8dff542817536040 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 9 Sep 2023 05:09:50 -0400
Subject: [PATCH 179/375] Update src/bisection.jl

---
 lib/SimpleNonlinearSolve/src/bisection.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 17836caf4..d0c134c04 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -25,7 +25,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : eps(1.0)
+    atol = abstol !== nothing ? abstol : minimum(eps(left), eps(right))
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,

From f5f61047cbb8cbc5e16562971d03d1c7ccdf585a Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 9 Sep 2023 05:10:34 -0400
Subject: [PATCH 180/375] Update src/bisection.jl

---
 lib/SimpleNonlinearSolve/src/bisection.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index d0c134c04..f1b69bd7b 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -25,7 +25,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : minimum(eps(left), eps(right))
+    atol = abstol !== nothing ? abstol : min(eps(left), eps(right))
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,

From 0a94645e7670ac580b9a6c70a62b801e21e4472c Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 9 Sep 2023 05:11:07 -0400
Subject: [PATCH 181/375] Update src/ridder.jl

---
 lib/SimpleNonlinearSolve/src/ridder.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index 190a37153..6e6e9464e 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -12,7 +12,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : minimum(eps(left), eps(right))
+    atol = abstol !== nothing ? abstol : min(eps(left), eps(right))
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;

From f1564c92973d1449a72d9db26ded8c99a9706f3a Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 9 Sep 2023 05:11:10 -0400
Subject: [PATCH 182/375] Update src/falsi.jl

---
 lib/SimpleNonlinearSolve/src/falsi.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index 549a52c13..52506f871 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -9,7 +9,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : minimum(eps(left), eps(right))
+    atol = abstol !== nothing ? abstol : min(eps(left), eps(right))
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;

From 4f89b5060f02ca61c55f57c9613c3c3c1decbeb5 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 9 Sep 2023 05:11:19 -0400
Subject: [PATCH 183/375] Update src/brent.jl

---
 lib/SimpleNonlinearSolve/src/brent.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index 1ddc0f348..11fcd5f34 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -13,7 +13,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
     a, b = prob.tspan
     fa, fb = f(a), f(b)
     ϵ = eps(convert(typeof(fa), 1.0))
-    atol = abstol !== nothing ? abstol : minimum(eps(left), eps(right))
+    atol = abstol !== nothing ? abstol : min(eps(left), eps(right))
 
     if iszero(fa)
         return SciMLBase.build_solution(prob, alg, a, fa;

From de98fe98a1dd57512d6748957d7d9001f9e4a7f7 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Mon, 11 Sep 2023 01:10:58 +0200
Subject: [PATCH 184/375] tol fix for itp

---
 lib/SimpleNonlinearSolve/src/brent.jl |  2 +-
 lib/SimpleNonlinearSolve/src/itp.jl   | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index 11fcd5f34..df2cd1d7d 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -13,7 +13,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
     a, b = prob.tspan
     fa, fb = f(a), f(b)
     ϵ = eps(convert(typeof(fa), 1.0))
-    atol = abstol !== nothing ? abstol : min(eps(left), eps(right))
+    atol = abstol !== nothing ? abstol : min(eps(a), eps(b))
 
     if iszero(fa)
         return SciMLBase.build_solution(prob, alg, a, fa;
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index fa390aa45..fa3bebe26 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -59,12 +59,12 @@ struct ITP{T} <: AbstractBracketingAlgorithm
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
-    args...; abstol = 1.0e-15,
+    args...; abstol = nothing,
     maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b
     fl, fr = f(left), f(right)
-    ϵ = abstol
+    ϵ = abstol !== nothing ? abstol : min(eps(left), eps(right))
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,
@@ -111,6 +111,12 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
             xp = mid - (σ * r)
         end
 
+        if abs((left - right) / 2) < ϵ
+            return SciMLBase.build_solution(prob, alg, mid, f(mid);
+            retcode = ReturnCode.Success,
+            left = left, right = right)
+        end
+    
         ## Update ##
         tmin, tmax = minmax(left, right)
         xp >= tmax && (xp = prevfloat(tmax))

From 8a32ee67c571029ba6eeb6c106ce7c78d3e8663c Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Mon, 11 Sep 2023 01:11:07 +0200
Subject: [PATCH 185/375] test for itp

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 23253240a..34468be58 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -386,6 +386,9 @@ for atol in tols
     sol = solve(probB, Falsi(), abstol = atol)
     @test abs(sol.u - sqrt(2)) < atol
     @test abs(sol.u - sqrt(2)) > ϵ
+    sol = solve(probB, ITP(), abstol = atol)
+    @test abs(sol.u - sqrt(2)) < atol
+    @test abs(sol.u - sqrt(2)) > ϵ
 end
 
 tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution

From 4402777519f9c1dd6469a09d8d923493a6824efd Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 12 Sep 2023 00:06:25 +0200
Subject: [PATCH 186/375] minor arg fix

---
 lib/SimpleNonlinearSolve/src/bisection.jl | 8 ++++----
 lib/SimpleNonlinearSolve/src/brent.jl     | 7 +++----
 lib/SimpleNonlinearSolve/src/falsi.jl     | 7 +++----
 lib/SimpleNonlinearSolve/src/itp.jl       | 4 ++--
 lib/SimpleNonlinearSolve/src/ridder.jl    | 7 +++----
 5 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index f1b69bd7b..d583bd35d 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -20,12 +20,12 @@ function Bisection(; exact_left = false, exact_right = false)
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...;
-    maxiters = 1000, abstol = nothing,
+    maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
     kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : min(eps(left), eps(right))
+    #atol = abstol
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,
@@ -46,7 +46,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
                     retcode = ReturnCode.FloatingPointLimit,
                     left = left, right = right)
             fm = f(mid)
-            if abs((right - left) / 2) < atol
+            if abs((right - left) / 2) < abstol
                 return SciMLBase.build_solution(prob, alg, mid, fm;
                     retcode = ReturnCode.Success,
                     left = left, right = right)
@@ -73,7 +73,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
                 retcode = ReturnCode.FloatingPointLimit,
                 left = left, right = right)
         fm = f(mid)
-        if abs((right - left) / 2) < atol
+        if abs((right - left) / 2) < abstol
             return SciMLBase.build_solution(prob, alg, mid, fm;
                 retcode = ReturnCode.Success,
                 left = left, right = right)
diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index df2cd1d7d..47e5495f0 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -7,13 +7,12 @@ A non-allocating Brent method
 struct Brent <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
-    maxiters = 1000, abstol = nothing,
+    maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
     kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     fa, fb = f(a), f(b)
     ϵ = eps(convert(typeof(fa), 1.0))
-    atol = abstol !== nothing ? abstol : min(eps(a), eps(b))
 
     if iszero(fa)
         return SciMLBase.build_solution(prob, alg, a, fa;
@@ -65,7 +64,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
                 cond = false
             end
             fs = f(s)
-            if abs((b - a) / 2) < atol
+            if abs((b - a) / 2) < abstol
                 return SciMLBase.build_solution(prob, alg, s, fs;
                     retcode = ReturnCode.Success,
                     left = a, right = b)
@@ -109,7 +108,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
                 left = a, right = b)
         end
         fc = f(c)
-        if abs((b - a) / 2) < atol
+        if abs((b - a) / 2) < abstol
             return SciMLBase.build_solution(prob, alg, c, fc;
                 retcode = ReturnCode.Success,
                 left = a, right = b)
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index 52506f871..de1079beb 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -4,12 +4,11 @@
 struct Falsi <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
-    maxiters = 1000, abstol = nothing,
+    maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
     kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : min(eps(left), eps(right))
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
@@ -37,7 +36,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
                 break
             end
             fm = f(mid)
-            if abs((right - left) / 2) < atol
+            if abs((right - left) / 2) < abstol
                 return SciMLBase.build_solution(prob, alg, mid, fm;
                     retcode = ReturnCode.Success,
                     left = left, right = right)
@@ -64,7 +63,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
                 retcode = ReturnCode.FloatingPointLimit,
                 left = left, right = right)
         fm = f(mid)
-        if abs((right - left) / 2) < atol
+        if abs((right - left) / 2) < abstol
             return SciMLBase.build_solution(prob, alg, mid, fm;
                 retcode = ReturnCode.Success,
                 left = left, right = right)
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index fa3bebe26..c5d87bbbe 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -59,12 +59,12 @@ struct ITP{T} <: AbstractBracketingAlgorithm
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
-    args...; abstol = nothing,
+    args...; abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
     maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b
     fl, fr = f(left), f(right)
-    ϵ = abstol !== nothing ? abstol : min(eps(left), eps(right))
+    ϵ = abstol
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,
diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index 6e6e9464e..ce95a178a 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -7,12 +7,11 @@ A non-allocating ridder method
 struct Ridder <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
-    maxiters = 1000, abstol = nothing,
+    maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
     kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    atol = abstol !== nothing ? abstol : min(eps(left), eps(right))
 
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
@@ -42,7 +41,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
             x = mid + (mid - left) * sign(fl - fr) * fm / s
             fx = f(x)
             xo = x
-            if abs((right - left) / 2) < atol
+            if abs((right - left) / 2) < abstol
                 return SciMLBase.build_solution(prob, alg, mid, fm;
                     retcode = ReturnCode.Success,
                     left = left, right = right)
@@ -76,7 +75,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
                 retcode = ReturnCode.FloatingPointLimit,
                 left = left, right = right)
         fm = f(mid)
-        if abs((right - left) / 2) < atol
+        if abs((right - left) / 2) < abstol
             return SciMLBase.build_solution(prob, alg, mid, fm;
                 retcode = ReturnCode.Success,
                 left = left, right = right)

From 35247e6daa819071fbd884cc96fa8e9e9b67e38e Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Tue, 12 Sep 2023 00:10:45 +0200
Subject: [PATCH 187/375] format

---
 lib/SimpleNonlinearSolve/src/bisection.jl | 1 -
 lib/SimpleNonlinearSolve/src/itp.jl       | 6 +++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index d583bd35d..24db4adcc 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -25,7 +25,6 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
-    #atol = abstol
     if iszero(fl)
         return SciMLBase.build_solution(prob, alg, left, fl;
             retcode = ReturnCode.ExactSolutionLeft, left = left,
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index c5d87bbbe..f6688381c 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -113,10 +113,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
 
         if abs((left - right) / 2) < ϵ
             return SciMLBase.build_solution(prob, alg, mid, f(mid);
-            retcode = ReturnCode.Success,
-            left = left, right = right)
+                retcode = ReturnCode.Success,
+                left = left, right = right)
         end
-    
+
         ## Update ##
         tmin, tmax = minmax(left, right)
         xp >= tmax && (xp = prevfloat(tmax))

From 09180cec58cacc2cd35ab9b1518ce20e94f21608 Mon Sep 17 00:00:00 2001
From: Yash Raj Singh <yash.raj270998@gmail.com>
Date: Fri, 15 Sep 2023 02:14:07 +0200
Subject: [PATCH 188/375] readme fix

---
 lib/SimpleNonlinearSolve/README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index 818872c0b..f6715e696 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -35,5 +35,7 @@ solver = solve(probN, SimpleNewtonRaphson(), tol = 1e-9)
 f(u, p) = u .* u .- 2.0
 u0 = (1.0, 2.0) # brackets
 probB = IntervalNonlinearProblem(f, u0)
-sol = solve(probB, Falsi())
+sol = solve(probB, ITP())
 ```
+
+For more details on the bracketing methods, refer to the [Tutorials](https://docs.sciml.ai/NonlinearSolve/stable/tutorials/nonlinear/#Using-Bracketing-Methods) and detailed [APIs](https://docs.sciml.ai/NonlinearSolve/stable/api/simplenonlinearsolve/#Solver-API)

From 8ea0f0efa7e43e54b93ecc2abbdfb3cd396c0138 Mon Sep 17 00:00:00 2001
From: CompatHelper Julia <compathelper_noreply@julialang.org>
Date: Fri, 22 Sep 2023 02:47:49 +0000
Subject: [PATCH 189/375] CompatHelper: bump compat for SciMLBase to 2, (keep
 existing compat)

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index ce68fc7b5..69e35abef 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -30,7 +30,7 @@ NNlib = "0.8, 0.9"
 PackageExtensionCompat = "1"
 PrecompileTools = "1"
 Reexport = "0.2, 1"
-SciMLBase = "1.73"
+SciMLBase = "1.73, 2"
 StaticArraysCore = "1.4"
 julia = "1.6"
 

From a6e6c0d9e1b2ba49a7727d806f77aac5c8fc11b2 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 22 Sep 2023 00:30:23 -0400
Subject: [PATCH 190/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 69e35abef..d72f9d000 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.19"
+version = "0.1.20"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

From c6a641376b40b00c97173b6039e24583ed44810e Mon Sep 17 00:00:00 2001
From: Tom Rottier <39778698+TomRottier@users.noreply.github.com>
Date: Mon, 16 Oct 2023 15:42:08 -0400
Subject: [PATCH 191/375] Fix error in README example

Used keyword tol
---
 lib/SimpleNonlinearSolve/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index f6715e696..c75b8ed62 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -28,7 +28,7 @@ using SimpleNonlinearSolve, StaticArrays
 f(u,p) = u .* u .- 2
 u0 = @SVector[1.0, 1.0]
 probN = NonlinearProblem{false}(f, u0)
-solver = solve(probN, SimpleNewtonRaphson(), tol = 1e-9)
+solver = solve(probN, SimpleNewtonRaphson(), abstol = 1e-9)
 
 ## Bracketing Methods
 

From 65bcea052a9fd20633fdbed6c1c43cb910fa8b4b Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Oct 2023 04:12:33 -0400
Subject: [PATCH 192/375] Set ITP as the default method for interval problems

---
 .../src/SimpleNonlinearSolve.jl                     | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 5c500d500..85052e5ff 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -46,6 +46,19 @@ include("batched/raphson.jl")
 include("batched/dfsane.jl")
 include("batched/broyden.jl")
 
+## Default algorithm
+
+# Set the default bracketing method to ITP
+
+function SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...)
+    SciMLBase.solve(prob, ITP(); kwargs...)
+end
+
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing,
+    args...; kwargs...)
+    SciMLBase.solve(prob, ITP(), args...; kwargs...)
+end
+
 import PrecompileTools
 
 PrecompileTools.@compile_workload begin

From e2f84c68dd0a8fddb3e744398cf28f9b8ee2a9dc Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Oct 2023 04:28:20 -0400
Subject: [PATCH 193/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index d72f9d000..11285a623 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.20"
+version = "0.1.21"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

From 1db1c982d612300fb471d62b1a9233337b50be85 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Oct 2023 06:11:55 -0400
Subject: [PATCH 194/375] Halley -> SimpleHalley

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 3 +--
 lib/SimpleNonlinearSolve/src/halley.jl               | 8 ++++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 85052e5ff..9c9c70fc6 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -88,8 +88,7 @@ PrecompileTools.@compile_workload begin
     end
 end
 
-# DiffEq styled algorithms
-export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, Halley, Klement,
+export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, SimpleHalley, Klement,
     Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld, ITP
 export BatchedBroyden, BatchedSimpleNewtonRaphson, BatchedSimpleDFSane
 
diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index 9e97a57cd..d307f99e9 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -1,6 +1,6 @@
 """
 ```julia
-Halley(; chunk_size = Val{0}(), autodiff = Val{true}(),
+SimpleHalley(; chunk_size = Val{0}(), autodiff = Val{true}(),
                                  diff_type = Val{:forward})
 ```
 
@@ -28,8 +28,8 @@ and static array problems.
   `Val{:forward}` for forward finite differences. For more details on the choices, see the
   [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
 """
-struct Halley{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
-    function Halley(; chunk_size = Val{0}(), autodiff = Val{true}(),
+struct SimpleHalley{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
+    function SimpleHalley(; chunk_size = Val{0}(), autodiff = Val{true}(),
         diff_type = Val{:forward})
         new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
             SciMLBase._unwrap_val(diff_type)}()
@@ -37,7 +37,7 @@ struct Halley{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-    alg::Halley, args...; abstol = nothing,
+    alg::SimpleHalley, args...; abstol = nothing,
     reltol = nothing,
     maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)

From 6d975f4391fb95a7a5a8c88763aafaf10bd73636 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Oct 2023 06:16:53 -0400
Subject: [PATCH 195/375] fix naming

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl |  4 ++--
 lib/SimpleNonlinearSolve/src/halley.jl               |  4 ++--
 lib/SimpleNonlinearSolve/test/basictests.jl          | 12 ++++++------
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 9c9c70fc6..c5632d7b8 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -36,7 +36,7 @@ include("ridder.jl")
 include("brent.jl")
 include("dfsane.jl")
 include("ad.jl")
-include("halley.jl")
+include("SimpleHalley.jl")
 include("alefeld.jl")
 include("itp.jl")
 
@@ -64,7 +64,7 @@ import PrecompileTools
 PrecompileTools.@compile_workload begin
     for T in (Float32, Float64)
         prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-        for alg in (SimpleNewtonRaphson, Halley, Broyden, Klement, SimpleTrustRegion,
+        for alg in (SimpleNewtonRaphson, SimpleHalley, Broyden, Klement, SimpleTrustRegion,
             SimpleDFSane)
             solve(prob_no_brack, alg(), abstol = T(1e-2))
         end
diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index d307f99e9..29cc1e54f 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -4,7 +4,7 @@ SimpleHalley(; chunk_size = Val{0}(), autodiff = Val{true}(),
                                  diff_type = Val{:forward})
 ```
 
-A low-overhead implementation of Halley's Method. This method is non-allocating on scalar
+A low-overhead implementation of SimpleHalley's Method. This method is non-allocating on scalar
 and static array problems.
 
 !!! note
@@ -49,7 +49,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
     T = typeof(x)
 
     if SciMLBase.isinplace(prob)
-        error("Halley currently only supports out-of-place nonlinear problems")
+        error("SimpleHalley currently only supports out-of-place nonlinear problems")
     end
 
     atol = abstol !== nothing ? abstol :
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 34468be58..027f766ed 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -56,10 +56,10 @@ if VERSION >= v"1.7"
     @test (@ballocated benchmark_scalar(sf, csu0)) == 0
 end
 
-# Halley
+# SimpleHalley
 function benchmark_scalar(f, u0)
     probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, Halley()))
+    sol = (solve(probN, SimpleHalley()))
 end
 
 function ff(u, p)
@@ -139,7 +139,7 @@ using ForwardDiff
 f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
 
 for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
-    SimpleDFSane(), Halley(), BROYDEN_SOLVERS...)
+    SimpleDFSane(), SimpleHalley(), BROYDEN_SOLVERS...)
     g = function (p)
         probN = NonlinearProblem{false}(f, csu0, p)
         sol = solve(probN, alg, abstol = 1e-9)
@@ -162,7 +162,7 @@ end
 # Scalar
 f, u0 = (u, p) -> u * u - p, 1.0
 for alg in (SimpleNewtonRaphson(), Klement(), SimpleTrustRegion(),
-    SimpleDFSane(), Halley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
+    SimpleDFSane(), SimpleHalley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
     g = function (p)
         probN = NonlinearProblem{false}(f, oftype(p, u0), p)
         sol = solve(probN, alg)
@@ -271,7 +271,7 @@ for alg in [Bisection(), Falsi(), Ridder(), Brent(), ITP()]
 end
 
 for alg in (SimpleNewtonRaphson(), Klement(), SimpleTrustRegion(),
-    SimpleDFSane(), Halley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
+    SimpleDFSane(), SimpleHalley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
     global g, p
     g = function (p)
         probN = NonlinearProblem{false}(f, 0.5, p)
@@ -288,7 +288,7 @@ probN = NonlinearProblem(f, u0)
 
 for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
     SimpleTrustRegion(),
-    SimpleTrustRegion(; autodiff = false), Halley(), Halley(; autodiff = false),
+    SimpleTrustRegion(; autodiff = false), SimpleHalley(), SimpleHalley(; autodiff = false),
     Klement(), SimpleDFSane(),
     BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
     sol = solve(probN, alg)

From ebb501f929c665aa9ff175ae521dcaacf8532fd3 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Oct 2023 06:19:41 -0400
Subject: [PATCH 196/375] fix page name

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index c5632d7b8..96d76d92e 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -36,7 +36,7 @@ include("ridder.jl")
 include("brent.jl")
 include("dfsane.jl")
 include("ad.jl")
-include("SimpleHalley.jl")
+include("halley.jl")
 include("alefeld.jl")
 include("itp.jl")
 

From d7d9c458dbc3ab8d645a199149e07442c7e93c49 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 17 Oct 2023 06:29:34 -0400
Subject: [PATCH 197/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 11285a623..61fe5f16a 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.21"
+version = "0.1.22"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

From 0f94ddadcb18be8c572cb29d7e2b1ba287b7a908 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 20 Oct 2023 08:08:08 -0400
Subject: [PATCH 198/375] Add matrix resizing and fix cases with u0 as a matrix

---
 lib/SimpleNonlinearSolve/Project.toml                 |  2 +-
 lib/SimpleNonlinearSolve/src/broyden.jl               |  6 +++---
 lib/SimpleNonlinearSolve/src/klement.jl               |  8 ++++----
 lib/SimpleNonlinearSolve/src/raphson.jl               |  2 +-
 lib/SimpleNonlinearSolve/src/utils.jl                 |  4 ++++
 .../test/matrix_resizing_tests.jl                     | 11 +++++++++++
 lib/SimpleNonlinearSolve/test/runtests.jl             | 10 +++-------
 7 files changed, 27 insertions(+), 16 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 61fe5f16a..28a6b0ed7 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.22"
+version = "0.1.23"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 6c5c3ce7f..2f518df83 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -58,12 +58,12 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
     xₙ₋₁ = x
     fₙ₋₁ = fₙ
     for _ in 1:maxiters
-        xₙ = xₙ₋₁ - J⁻¹ * fₙ₋₁
+        xₙ = xₙ₋₁ - ArrayInterface.restructure(xₙ₋₁, J⁻¹ * _vec(fₙ₋₁))
         fₙ = f(xₙ)
         Δxₙ = xₙ - xₙ₋₁
         Δfₙ = fₙ - fₙ₋₁
-        J⁻¹Δfₙ = J⁻¹ * Δfₙ
-        J⁻¹ += ((Δxₙ .- J⁻¹Δfₙ) ./ (Δxₙ' * J⁻¹Δfₙ)) * (Δxₙ' * J⁻¹)
+        J⁻¹Δfₙ = ArrayInterface.restructure(Δfₙ, J⁻¹ * _vec(Δfₙ))
+        J⁻¹ += ArrayInterface.restructure(J⁻¹, ((_vec(Δxₙ) .- _vec(J⁻¹Δfₙ)) ./ (_vec(Δxₙ)' * _vec(J⁻¹Δfₙ))) * (_vec(Δxₙ)' * J⁻¹))
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
             return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 00264d32f..7235487c5 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -75,7 +75,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                 F = lu(J, check = false)
             end
 
-            tmp = F \ fₙ₋₁
+            tmp = ArrayInterface.restructure(fₙ₋₁, F \ _vec(fₙ₋₁))
             xₙ = xₙ₋₁ - tmp
             fₙ = f(xₙ)
 
@@ -92,10 +92,10 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             Δfₙ = fₙ - fₙ₋₁
 
             # Prevent division by 0
-            denominator = max.(J' .^ 2 * Δxₙ .^ 2, 1e-9)
+            denominator = ArrayInterface.restructure(Δxₙ, max.(J' .^ 2 * _vec(Δxₙ) .^ 2, 1e-9))
 
-            k = (Δfₙ - J * Δxₙ) ./ denominator
-            J += (k * Δxₙ' .* J) * J
+            k = (Δfₙ - ArrayInterface.restructure(Δxₙ, J * _vec(Δxₙ))) ./ denominator
+            J += (_vec(k) * _vec(Δxₙ)' .* J) * J
 
             xₙ₋₁ = xₙ
             fₙ₋₁ = fₙ
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 48b8f7591..a7d9858bb 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -100,7 +100,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         end
         iszero(fx) &&
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-        Δx = dfx \ fx
+        Δx = ArrayInterface.restructure(fx, dfx \ _vec(fx))
         x -= Δx
         if isapprox(x, xo, atol = atol, rtol = rtol)
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 12462a05c..4f5617786 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -82,3 +82,7 @@ function dogleg_method(H, g, Δ)
     tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd
     return δsd + tau * δN_δsd
 end
+
+@inline _vec(v) = vec(v)
+@inline _vec(v::Number) = v
+@inline _vec(v::AbstractVector) = v
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
new file mode 100644
index 000000000..9612cbb68
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
@@ -0,0 +1,11 @@
+using SimpleNonlinearSolve
+
+ff(u, p) = u .* u .- p
+u0 = rand(2,2)
+p = 2.0
+vecprob = NonlinearProblem(ff, vec(u0), p)
+prob = NonlinearProblem(ff, u0, p)
+
+for alg in (Klement(), Broyden(), SimpleNewtonRaphson())
+    @test vec(solve(prob, alg).u) == solve(vecprob, alg).u
+end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index bea57ea0a..98a01bdba 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -4,12 +4,8 @@ const GROUP = get(ENV, "GROUP", "All")
 
 @time begin
     if GROUP == "All" || GROUP == "Core"
-        @time @safetestset "Basic Tests + Some AD" begin
-            include("basictests.jl")
-        end
-
-        @time @safetestset "Inplace Tests" begin
-            include("inplace.jl")
-        end
+        @time @safetestset "Basic Tests + Some AD" include("basictests.jl")
+        @time @safetestset "Inplace Tests" include("inplace.jl")
+        @time @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
     end
 end

From 580c439b58999e6552a25e1236836e539eccf986 Mon Sep 17 00:00:00 2001
From: Chris Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 20 Oct 2023 08:57:38 -0400
Subject: [PATCH 199/375] don't restructure on number

---
 lib/SimpleNonlinearSolve/src/broyden.jl | 6 +++---
 lib/SimpleNonlinearSolve/src/klement.jl | 6 +++---
 lib/SimpleNonlinearSolve/src/raphson.jl | 2 +-
 lib/SimpleNonlinearSolve/src/utils.jl   | 5 ++++-
 4 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 2f518df83..9f3c22505 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -58,12 +58,12 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
     xₙ₋₁ = x
     fₙ₋₁ = fₙ
     for _ in 1:maxiters
-        xₙ = xₙ₋₁ - ArrayInterface.restructure(xₙ₋₁, J⁻¹ * _vec(fₙ₋₁))
+        xₙ = xₙ₋₁ - _restructure(xₙ₋₁, J⁻¹ * _vec(fₙ₋₁))
         fₙ = f(xₙ)
         Δxₙ = xₙ - xₙ₋₁
         Δfₙ = fₙ - fₙ₋₁
-        J⁻¹Δfₙ = ArrayInterface.restructure(Δfₙ, J⁻¹ * _vec(Δfₙ))
-        J⁻¹ += ArrayInterface.restructure(J⁻¹, ((_vec(Δxₙ) .- _vec(J⁻¹Δfₙ)) ./ (_vec(Δxₙ)' * _vec(J⁻¹Δfₙ))) * (_vec(Δxₙ)' * J⁻¹))
+        J⁻¹Δfₙ = _restructure(Δfₙ, J⁻¹ * _vec(Δfₙ))
+        J⁻¹ += _restructure(J⁻¹, ((_vec(Δxₙ) .- _vec(J⁻¹Δfₙ)) ./ (_vec(Δxₙ)' * _vec(J⁻¹Δfₙ))) * (_vec(Δxₙ)' * J⁻¹))
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
             return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 7235487c5..799ba5987 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -75,7 +75,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                 F = lu(J, check = false)
             end
 
-            tmp = ArrayInterface.restructure(fₙ₋₁, F \ _vec(fₙ₋₁))
+            tmp = _restructure(fₙ₋₁, F \ _vec(fₙ₋₁))
             xₙ = xₙ₋₁ - tmp
             fₙ = f(xₙ)
 
@@ -92,9 +92,9 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             Δfₙ = fₙ - fₙ₋₁
 
             # Prevent division by 0
-            denominator = ArrayInterface.restructure(Δxₙ, max.(J' .^ 2 * _vec(Δxₙ) .^ 2, 1e-9))
+            denominator = _restructure(Δxₙ, max.(J' .^ 2 * _vec(Δxₙ) .^ 2, 1e-9))
 
-            k = (Δfₙ - ArrayInterface.restructure(Δxₙ, J * _vec(Δxₙ))) ./ denominator
+            k = (Δfₙ - _restructure(Δxₙ, J * _vec(Δxₙ))) ./ denominator
             J += (_vec(k) * _vec(Δxₙ)' .* J) * J
 
             xₙ₋₁ = xₙ
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index a7d9858bb..c36dc3504 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -100,7 +100,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         end
         iszero(fx) &&
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-        Δx = ArrayInterface.restructure(fx, dfx \ _vec(fx))
+        Δx = _restructure(fx, dfx \ _vec(fx))
         x -= Δx
         if isapprox(x, xo, atol = atol, rtol = rtol)
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 4f5617786..c0245ece0 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -85,4 +85,7 @@ end
 
 @inline _vec(v) = vec(v)
 @inline _vec(v::Number) = v
-@inline _vec(v::AbstractVector) = v
\ No newline at end of file
+@inline _vec(v::AbstractVector) = v
+
+@inline _restructure(y::Number, x::Number) = x
+@inline _restructure(y, x) = ArrayInterface.restructure(y,x)
\ No newline at end of file

From 0e6aba39e0b82b2a1cedd42f8ae315236b3837ad Mon Sep 17 00:00:00 2001
From: ChrisRackauckas <ChrisRackauckas@users.noreply.github.com>
Date: Sat, 21 Oct 2023 00:18:21 +0000
Subject: [PATCH 200/375] Format .jl files

---
 lib/SimpleNonlinearSolve/src/broyden.jl                | 4 +++-
 lib/SimpleNonlinearSolve/src/utils.jl                  | 2 +-
 lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl | 2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 9f3c22505..045345f29 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -63,7 +63,9 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
         Δxₙ = xₙ - xₙ₋₁
         Δfₙ = fₙ - fₙ₋₁
         J⁻¹Δfₙ = _restructure(Δfₙ, J⁻¹ * _vec(Δfₙ))
-        J⁻¹ += _restructure(J⁻¹, ((_vec(Δxₙ) .- _vec(J⁻¹Δfₙ)) ./ (_vec(Δxₙ)' * _vec(J⁻¹Δfₙ))) * (_vec(Δxₙ)' * J⁻¹))
+        J⁻¹ += _restructure(J⁻¹,
+            ((_vec(Δxₙ) .- _vec(J⁻¹Δfₙ)) ./ (_vec(Δxₙ)' * _vec(J⁻¹Δfₙ))) *
+            (_vec(Δxₙ)' * J⁻¹))
 
         if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
             return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index c0245ece0..1c4400026 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -88,4 +88,4 @@ end
 @inline _vec(v::AbstractVector) = v
 
 @inline _restructure(y::Number, x::Number) = x
-@inline _restructure(y, x) = ArrayInterface.restructure(y,x)
\ No newline at end of file
+@inline _restructure(y, x) = ArrayInterface.restructure(y, x)
diff --git a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
index 9612cbb68..9a1989b71 100644
--- a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
@@ -1,7 +1,7 @@
 using SimpleNonlinearSolve
 
 ff(u, p) = u .* u .- p
-u0 = rand(2,2)
+u0 = rand(2, 2)
 p = 2.0
 vecprob = NonlinearProblem(ff, vec(u0), p)
 prob = NonlinearProblem(ff, u0, p)

From cca90d051fa0dff9a57825cb0fe1b74a68c2a00e Mon Sep 17 00:00:00 2001
From: Pepijn de Vos <pepijn.devos@juliacomputing.com>
Date: Thu, 2 Nov 2023 18:21:32 +0100
Subject: [PATCH 201/375] Change typeof(x) <: y to x isa y

---
 lib/SimpleNonlinearSolve/src/halley.jl  | 2 +-
 lib/SimpleNonlinearSolve/src/raphson.jl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index 29cc1e54f..2c0496cfe 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -56,7 +56,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
     rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
 
-    if typeof(x) <: Number
+    if x isa Number
         xo = oftype(one(eltype(x)), Inf)
     else
         xo = map(x -> oftype(one(eltype(x)), Inf), x)
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index c36dc3504..6af24b733 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -78,7 +78,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
     rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
 
-    if typeof(x) <: Number
+    if x isa Number
         xo = oftype(one(eltype(x)), Inf)
     else
         xo = map(x -> oftype(one(eltype(x)), Inf), x)

From ad12493fdb3ca8ee7dbfed3ac8588ca41b4d1433 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 30 Oct 2023 04:29:54 -0400
Subject: [PATCH 202/375] Add a dispatch to SimpleNewtonRaphson for NNLS and
 SimpleGaussNewton

Fixes https://github.com/SciML/SimpleNonlinearSolve.jl/issues/82
---
 .../src/SimpleNonlinearSolve.jl               |  2 +-
 lib/SimpleNonlinearSolve/src/raphson.jl       | 16 ++++++++++++++--
 .../test/least_squares.jl                     | 19 +++++++++++++++++++
 lib/SimpleNonlinearSolve/test/runtests.jl     |  1 +
 4 files changed, 35 insertions(+), 3 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/test/least_squares.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 96d76d92e..0713ea2c6 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -89,7 +89,7 @@ PrecompileTools.@compile_workload begin
 end
 
 export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, SimpleHalley, Klement,
-    Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld, ITP
+    Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld, ITP, SimpleGaussNewton
 export BatchedBroyden, BatchedSimpleNewtonRaphson, BatchedSimpleDFSane
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 6af24b733..d5bad8d13 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -61,7 +61,9 @@ function SimpleNewtonRaphson(; batched = false,
         SciMLBase._unwrap_val(diff_type)}()
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem,
+const SimpleGaussNewton = SimpleNewtonRaphson
+
+function SciMLBase.__solve(prob::Union{NonlinearProblem,NonlinearLeastSquaresProblem},
     alg::SimpleNewtonRaphson, args...; abstol = nothing,
     reltol = nothing,
     maxiters = 1000, kwargs...)
@@ -74,6 +76,10 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         error("SimpleNewtonRaphson currently only supports out-of-place nonlinear problems")
     end
 
+    if prob isa NonlinearLeastSquaresProblem && !(typeof(prob.u0) <: Union{Number, AbstractVector})
+        error("SimpleGaussNewton only supports Number and AbstactVector types. Please convert any problem of AbstractArray into one with u0 as AbstractVector")
+    end
+
     atol = abstol !== nothing ? abstol :
            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
     rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
@@ -100,7 +106,13 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         end
         iszero(fx) &&
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-        Δx = _restructure(fx, dfx \ _vec(fx))
+
+        if prob isa NonlinearProblem
+            Δx = _restructure(fx, dfx \ _vec(fx))
+        else
+            Δx = dfx \ fx
+        end
+
         x -= Δx
         if isapprox(x, xo, atol = atol, rtol = rtol)
             return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/test/least_squares.jl b/lib/SimpleNonlinearSolve/test/least_squares.jl
new file mode 100644
index 000000000..64b656e8c
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/least_squares.jl
@@ -0,0 +1,19 @@
+using SimpleNonlinearSolve, Random, LinearAlgebra, Test
+
+true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
+
+θ_true = [1.0, 0.1, 2.0, 0.5]
+x = [-1.0, -0.5, 0.0, 0.5, 1.0]
+y_target = true_function(x, θ_true)
+
+function loss_function(θ, p)
+    ŷ = true_function(p, θ)
+    return abs2.(ŷ .- y_target)
+end
+
+θ_init = θ_true .+ randn!(similar(θ_true)) * 0.1
+prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
+sol = solve(prob_oop, SimpleNewtonRaphson())
+sol = solve(prob_oop, SimpleGaussNewton())
+
+@test norm(sol.resid) < 1e-12
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 98a01bdba..0c2a3dfde 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -7,5 +7,6 @@ const GROUP = get(ENV, "GROUP", "All")
         @time @safetestset "Basic Tests + Some AD" include("basictests.jl")
         @time @safetestset "Inplace Tests" include("inplace.jl")
         @time @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
+        @time @safetestset "Least Squares Tests" include("least_sqaures.jl")
     end
 end

From 754b15e1ad37885e3c06a7fb2f85a55d8ad5d66e Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 3 Nov 2023 10:05:30 -0400
Subject: [PATCH 203/375] Update runtests.jl

---
 lib/SimpleNonlinearSolve/test/runtests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 0c2a3dfde..d0fd1ff9b 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -7,6 +7,6 @@ const GROUP = get(ENV, "GROUP", "All")
         @time @safetestset "Basic Tests + Some AD" include("basictests.jl")
         @time @safetestset "Inplace Tests" include("inplace.jl")
         @time @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
-        @time @safetestset "Least Squares Tests" include("least_sqaures.jl")
+        @time @safetestset "Least Squares Tests" include("least_squares.jl")
     end
 end

From f40bc46ebd31a71ba0e727b1bdd8c5808a342045 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 3 Nov 2023 10:16:49 -0400
Subject: [PATCH 204/375] Update test/least_squares.jl

---
 lib/SimpleNonlinearSolve/test/least_squares.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/least_squares.jl b/lib/SimpleNonlinearSolve/test/least_squares.jl
index 64b656e8c..1e0f135ef 100644
--- a/lib/SimpleNonlinearSolve/test/least_squares.jl
+++ b/lib/SimpleNonlinearSolve/test/least_squares.jl
@@ -11,7 +11,7 @@ function loss_function(θ, p)
     return abs2.(ŷ .- y_target)
 end
 
-θ_init = θ_true .+ randn!(similar(θ_true)) * 0.1
+θ_init = θ_true .+ 0.1
 prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
 sol = solve(prob_oop, SimpleNewtonRaphson())
 sol = solve(prob_oop, SimpleGaussNewton())

From 27af8bdd317c00680427d24e9883892247fae96e Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 3 Nov 2023 10:44:46 -0400
Subject: [PATCH 205/375] Update least_squares.jl

---
 lib/SimpleNonlinearSolve/test/least_squares.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/least_squares.jl b/lib/SimpleNonlinearSolve/test/least_squares.jl
index 1e0f135ef..0b6647372 100644
--- a/lib/SimpleNonlinearSolve/test/least_squares.jl
+++ b/lib/SimpleNonlinearSolve/test/least_squares.jl
@@ -1,4 +1,4 @@
-using SimpleNonlinearSolve, Random, LinearAlgebra, Test
+using SimpleNonlinearSolve, LinearAlgebra, Test
 
 true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
 
@@ -16,4 +16,4 @@ prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
 sol = solve(prob_oop, SimpleNewtonRaphson())
 sol = solve(prob_oop, SimpleGaussNewton())
 
-@test norm(sol.resid) < 1e-12
\ No newline at end of file
+@test norm(sol.resid) < 1e-12

From cdbc5674bb388ba5e41177117d4730009a97b0d1 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 3 Nov 2023 11:06:24 -0400
Subject: [PATCH 206/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 28a6b0ed7..baf991736 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.23"
+version = "0.1.24"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

From 2292fbe7a072547b6f89ea6498f399f9b5ac2c92 Mon Sep 17 00:00:00 2001
From: Hendrik Ranocha <ranocha@users.noreply.github.com>
Date: Fri, 3 Nov 2023 16:44:50 +0100
Subject: [PATCH 207/375] fix badges in README.md

---
 lib/SimpleNonlinearSolve/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index c75b8ed62..53b32e311 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -3,8 +3,8 @@
 [![Join the chat at https://julialang.zulipchat.com #sciml-bridged](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/279055-sciml-bridged)
 [![Global Docs](https://img.shields.io/badge/docs-SciML-blue.svg)](https://docs.sciml.ai/NonlinearSolve/stable/)
 
-[![codecov](https://codecov.io/gh/SciML/NonlinearSolve.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/SciML/NonlinearSolve.jl)
-[![Build Status](https://github.com/SciML/NonlinearSolve.jl/workflows/CI/badge.svg)](https://github.com/SciML/NonlinearSolve.jl/actions?query=workflow%3ACI)
+[![codecov](https://codecov.io/gh/SciML/SimpleNonlinearSolve.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/SciML/SimpleNonlinearSolve.jl)
+[![Build Status](https://github.com/SciML/SimpleNonlinearSolve.jl/workflows/CI/badge.svg)](https://github.com/SciML/SimpleNonlinearSolve.jl/actions?query=workflow%3ACI)
 
 [![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac)
 [![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle)

From c6ee5418d111b227d76bbf7261879803f29a1840 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sun, 5 Nov 2023 18:43:11 -0500
Subject: [PATCH 208/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index baf991736..acad73005 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -26,6 +26,7 @@ ArrayInterface = "6, 7"
 DiffEqBase = "6.126"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
+LinearAlgebra = "1.6"
 NNlib = "0.8, 0.9"
 PackageExtensionCompat = "1"
 PrecompileTools = "1"

From 77c359fd37ce6f999d85a3dab966ab669408ee19 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 7 Nov 2023 11:15:24 +0100
Subject: [PATCH 209/375] Require v1.9 and drop backwards compat

---
 lib/SimpleNonlinearSolve/.github/workflows/CI.yml  |  1 -
 .../.github/workflows/Downstream.yml               |  2 +-
 lib/SimpleNonlinearSolve/Project.toml              | 14 ++++++--------
 .../src/SimpleNonlinearSolve.jl                    |  5 -----
 4 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index 1a6859e3a..ba2974f47 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -15,7 +15,6 @@ jobs:
           - Core
         version:
           - '1'
-          - '1.6'
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
index 4b1032411..affe2fd97 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        julia-version: [1,1.6]
+        julia-version: [1]
         os: [ubuntu-latest]
         package:
           - {user: SciML, repo: ModelingToolkit.jl, group: All}
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index acad73005..86ae7359e 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.24"
+version = "0.1.25"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
@@ -9,7 +9,6 @@ DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-PackageExtensionCompat = "65ce6f38-6b18-4e1d-a461-8949797d7930"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
@@ -22,18 +21,17 @@ NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 SimpleNonlinearSolveNNlibExt = "NNlib"
 
 [compat]
-ArrayInterface = "6, 7"
+ArrayInterface = "7"
 DiffEqBase = "6.126"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
-LinearAlgebra = "1.6"
+LinearAlgebra = "1.9"
 NNlib = "0.8, 0.9"
-PackageExtensionCompat = "1"
 PrecompileTools = "1"
-Reexport = "0.2, 1"
-SciMLBase = "1.73, 2"
+Reexport = "1"
+SciMLBase = "2"
 StaticArraysCore = "1.4"
-julia = "1.6"
+julia = "1.9"
 
 [extras]
 NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 0713ea2c6..cfd73e3b1 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -10,11 +10,6 @@ using DiffEqBase
 
 @reexport using SciMLBase
 
-using PackageExtensionCompat
-function __init__()
-    @require_extensions
-end
-
 const NNlibExtLoaded = Ref{Bool}(false)
 
 abstract type AbstractSimpleNonlinearSolveAlgorithm <: SciMLBase.AbstractNonlinearAlgorithm end

From 960ed93d4606bd391bfcddbde2c953c2abf8d853 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 7 Nov 2023 14:02:43 +0100
Subject: [PATCH 210/375] Require SciMLBase with v2.7 for NonlinearLeastSquares

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 86ae7359e..aa45b60af 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -29,7 +29,7 @@ LinearAlgebra = "1.9"
 NNlib = "0.8, 0.9"
 PrecompileTools = "1"
 Reexport = "1"
-SciMLBase = "2"
+SciMLBase = "2.7"
 StaticArraysCore = "1.4"
 julia = "1.9"
 

From 7f16738609088005891f5dd05f7bd56939e5266c Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 7 Nov 2023 14:25:32 +0100
Subject: [PATCH 211/375] Update least_squares.jl

Fix residual definition in test
---
 lib/SimpleNonlinearSolve/test/least_squares.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/least_squares.jl b/lib/SimpleNonlinearSolve/test/least_squares.jl
index 0b6647372..a7003f697 100644
--- a/lib/SimpleNonlinearSolve/test/least_squares.jl
+++ b/lib/SimpleNonlinearSolve/test/least_squares.jl
@@ -8,7 +8,7 @@ y_target = true_function(x, θ_true)
 
 function loss_function(θ, p)
     ŷ = true_function(p, θ)
-    return abs2.(ŷ .- y_target)
+    return ŷ .- y_target
 end
 
 θ_init = θ_true .+ 0.1

From 9387e5f09d8df8d676a685561010de23d9d587f4 Mon Sep 17 00:00:00 2001
From: FHoltorf <holtorf93@gmail.com>
Date: Sat, 18 Nov 2023 11:32:54 -0500
Subject: [PATCH 212/375] fixed gradient and GN Hessian approximation

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 1423d59b8..0a97e7d92 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -134,8 +134,8 @@ function SciMLBase.__solve(prob::NonlinearProblem,
     end
 
     fₖ = 0.5 * norm(F)^2
-    H = ∇f * ∇f
-    g = ∇f * F
+    H = ∇f' * ∇f
+    g = ∇f' * F
     shrink_counter = 0
 
     for k in 1:maxiters
@@ -188,8 +188,8 @@ function SciMLBase.__solve(prob::NonlinearProblem,
                 Δ = min(t₂ * Δ, Δₘₐₓ)
             end
             fₖ = fₖ₊₁
-            H = ∇f * ∇f
-            g = ∇f * F
+            H = ∇f' * ∇f
+            g = ∇f' * F
         end
     end
     return SciMLBase.build_solution(prob, alg, x, F; retcode = ReturnCode.MaxIters)

From 5b9100a8358efcf48eadcc0fe3e660e9861b9dd0 Mon Sep 17 00:00:00 2001
From: FHoltorf <holtorf93@gmail.com>
Date: Sat, 18 Nov 2023 11:33:55 -0500
Subject: [PATCH 213/375] make step accepteance less consersative for better
 performance

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 0a97e7d92..f1e992572 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -68,7 +68,7 @@ struct SimpleTrustRegion{T, CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
         diff_type = Val{:forward},
         max_trust_radius::Real = 0.0,
         initial_trust_radius::Real = 0.0,
-        step_threshold::Real = 0.1,
+        step_threshold::Real = 0.0001,
         shrink_threshold::Real = 0.25,
         expand_threshold::Real = 0.75,
         shrink_factor::Real = 0.25,

From 7c6b5724b6c9c68629d718bfc72c1a7b4a9c471f Mon Sep 17 00:00:00 2001
From: FHoltorf <holtorf93@gmail.com>
Date: Tue, 21 Nov 2023 22:19:30 -0500
Subject: [PATCH 214/375] simple fraction with forward slash instead of
 backslash (for otherwise rather subtle)

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index f1e992572..ab28e1ce3 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -147,7 +147,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
 
         # Compute the ratio of the actual to predicted reduction.
         model = -(δ' * g + 0.5 * δ' * H * δ)
-        r = model \ (fₖ - fₖ₊₁)
+        r = (fₖ - fₖ₊₁) / model 
 
         # Update the trust region radius.
         if r < η₂

From 66a073214d2fa4b3786f3310536ce92b2fc7a277 Mon Sep 17 00:00:00 2001
From: FHoltorf <holtorf93@gmail.com>
Date: Tue, 21 Nov 2023 22:20:46 -0500
Subject: [PATCH 215/375] run formatter

---
 .../ext/SimpleNonlinearSolveNNlibExt.jl       | 10 +++---
 .../src/SimpleNonlinearSolve.jl               |  2 +-
 lib/SimpleNonlinearSolve/src/ad.jl            | 36 +++++++++----------
 lib/SimpleNonlinearSolve/src/alefeld.jl       |  6 ++--
 .../src/batched/dfsane.jl                     | 12 +++----
 .../src/batched/raphson.jl                    | 12 +++----
 lib/SimpleNonlinearSolve/src/batched/utils.jl |  6 ++--
 lib/SimpleNonlinearSolve/src/bisection.jl     |  4 +--
 lib/SimpleNonlinearSolve/src/brent.jl         |  4 +--
 lib/SimpleNonlinearSolve/src/broyden.jl       |  8 ++---
 lib/SimpleNonlinearSolve/src/dfsane.jl        | 18 +++++-----
 lib/SimpleNonlinearSolve/src/falsi.jl         |  4 +--
 lib/SimpleNonlinearSolve/src/halley.jl        |  8 ++---
 lib/SimpleNonlinearSolve/src/itp.jl           |  4 +--
 lib/SimpleNonlinearSolve/src/klement.jl       |  6 ++--
 lib/SimpleNonlinearSolve/src/lbroyden.jl      | 18 +++++-----
 lib/SimpleNonlinearSolve/src/raphson.jl       | 19 +++++-----
 lib/SimpleNonlinearSolve/src/ridder.jl        |  4 +--
 lib/SimpleNonlinearSolve/src/trustRegion.jl   | 28 +++++++--------
 lib/SimpleNonlinearSolve/src/utils.jl         |  8 ++---
 lib/SimpleNonlinearSolve/test/inplace.jl      |  4 +--
 21 files changed, 111 insertions(+), 110 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
index 5b06530a6..1132b64b7 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
@@ -10,11 +10,11 @@ function __init__()
 end
 
 @views function SciMLBase.__solve(prob::NonlinearProblem,
-    alg::BatchedBroyden;
-    abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000,
-    kwargs...)
+        alg::BatchedBroyden;
+        abstol = nothing,
+        reltol = nothing,
+        maxiters = 1000,
+        kwargs...)
     iip = isinplace(prob)
 
     u, f, reconstruct = _construct_batched_problem_structure(prob)
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index cfd73e3b1..8c84c4377 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -50,7 +50,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...)
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing,
-    args...; kwargs...)
+        args...; kwargs...)
     SciMLBase.solve(prob, ITP(), args...; kwargs...)
 end
 
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 009cd227b..b0fd9f11c 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -29,19 +29,19 @@ function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
 end
 
 function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
-        iip,
-        <:Dual{T, V, P}},
-    alg::AbstractSimpleNonlinearSolveAlgorithm,
-    args...; kwargs...) where {iip, T, V, P}
+            iip,
+            <:Dual{T, V, P}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...; kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
         retcode = sol.retcode)
 end
 function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
-        iip,
-        <:AbstractArray{<:Dual{T, V, P}}},
-    alg::AbstractSimpleNonlinearSolveAlgorithm, args...;
-    kwargs...) where {iip, T, V, P}
+            iip,
+            <:AbstractArray{<:Dual{T, V, P}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm, args...;
+        kwargs...) where {iip, T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
         retcode = sol.retcode)
@@ -50,9 +50,9 @@ end
 # avoid ambiguities
 for Alg in [Bisection]
     @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
-            <:Dual{T, V, P}},
-        alg::$Alg, args...;
-        kwargs...) where {uType, iip, T, V, P}
+                <:Dual{T, V, P}},
+            alg::$Alg, args...;
+            kwargs...) where {uType, iip, T, V, P}
         sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
         return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials),
             sol.resid; retcode = sol.retcode,
@@ -61,13 +61,13 @@ for Alg in [Bisection]
         #return BracketingSolution(Dual{T,V,P}(sol.left, partials), Dual{T,V,P}(sol.right, partials), sol.retcode, sol.resid)
     end
     @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
-            <:AbstractArray{
-                <:Dual{T,
-                    V,
-                    P},
-            }},
-        alg::$Alg, args...;
-        kwargs...) where {uType, iip, T, V, P}
+                <:AbstractArray{
+                    <:Dual{T,
+                        V,
+                        P},
+                }},
+            alg::$Alg, args...;
+            kwargs...) where {uType, iip, T, V, P}
         sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
         return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials),
             sol.resid; retcode = sol.retcode,
diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index a2669ca96..0d4f56116 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -9,9 +9,9 @@ algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal
 struct Alefeld <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem,
-    alg::Alefeld, args...; abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000, kwargs...)
+        alg::Alefeld, args...; abstol = nothing,
+        reltol = nothing,
+        maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     c = a - (b - a) / (f(b) - f(a)) * f(a)
diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
index 60bb6ae12..01b3b1996 100644
--- a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
@@ -16,12 +16,12 @@ Base.@kwdef struct BatchedSimpleDFSane{T, F, TC <: NLSolveTerminationCondition}
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-    alg::BatchedSimpleDFSane,
-    args...;
-    abstol = nothing,
-    reltol = nothing,
-    maxiters = 100,
-    kwargs...)
+        alg::BatchedSimpleDFSane,
+        args...;
+        abstol = nothing,
+        reltol = nothing,
+        maxiters = 100,
+        kwargs...)
     iip = isinplace(prob)
 
     u, f, reconstruct = _construct_batched_problem_structure(prob)
diff --git a/lib/SimpleNonlinearSolve/src/batched/raphson.jl b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
index a141819bc..7bc7b8c4a 100644
--- a/lib/SimpleNonlinearSolve/src/batched/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
@@ -7,18 +7,18 @@ alg_autodiff(alg::BatchedSimpleNewtonRaphson{CS, AD, FDT}) where {CS, AD, FDT} =
 diff_type(alg::BatchedSimpleNewtonRaphson{CS, AD, FDT}) where {CS, AD, FDT} = FDT
 
 function BatchedSimpleNewtonRaphson(; chunk_size = Val{0}(),
-    autodiff = Val{true}(),
-    diff_type = Val{:forward},
-    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-        abstol = nothing,
-        reltol = nothing))
+        autodiff = Val{true}(),
+        diff_type = Val{:forward},
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing))
     return BatchedSimpleNewtonRaphson{SciMLBase._unwrap_val(chunk_size),
         SciMLBase._unwrap_val(autodiff),
         SciMLBase._unwrap_val(diff_type), typeof(termination_condition)}(termination_condition)
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::BatchedSimpleNewtonRaphson;
-    abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
+        abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
     iip = SciMLBase.isinplace(prob)
     iip &&
         @assert alg_autodiff(alg) "Inplace BatchedSimpleNewtonRaphson currently only supports autodiff."
diff --git a/lib/SimpleNonlinearSolve/src/batched/utils.jl b/lib/SimpleNonlinearSolve/src/batched/utils.jl
index 7b85011f1..b8e66fe80 100644
--- a/lib/SimpleNonlinearSolve/src/batched/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/batched/utils.jl
@@ -27,9 +27,9 @@ function _construct_batched_problem_structure(prob)
 end
 
 function _construct_batched_problem_structure(u0::AbstractArray{T, N},
-    f,
-    p,
-    ::Val{iip}) where {T, N, iip}
+        f,
+        p,
+        ::Val{iip}) where {T, N, iip}
     # Reconstruct `u`
     reconstruct = N == 2 ? identity : Base.Fix2(reshape, size(u0))
     # Standardize `u`
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 24db4adcc..f7c98aa65 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -20,8 +20,8 @@ function Bisection(; exact_left = false, exact_right = false)
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...;
-    maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-    kwargs...)
+        maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
+        kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index 47e5495f0..7d7a6bcf9 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -7,8 +7,8 @@ A non-allocating Brent method
 struct Brent <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
-    maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-    kwargs...)
+        maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
+        kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     fa, fb = f(a), f(b)
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 045345f29..07b2609f9 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -18,9 +18,9 @@ struct Broyden{TC <: NLSolveTerminationCondition} <:
 end
 
 function Broyden(; batched = false,
-    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-        abstol = nothing,
-        reltol = nothing))
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing))
     if batched
         @assert NNlibExtLoaded[] "Please install and load `NNlib.jl` to use batched Broyden."
         return BatchedBroyden(termination_condition)
@@ -29,7 +29,7 @@ function Broyden(; batched = false,
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
-    abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
+        abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
     f = Base.Fix2(prob.f, prob.p)
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index 2e52cde4f..49c50bca3 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -65,13 +65,13 @@ struct SimpleDFSane{T, TC} <: AbstractSimpleNonlinearSolveAlgorithm
 end
 
 function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
-    M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
-    nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2,
-    termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-        abstol = nothing,
-        reltol = nothing),
-    batched::Bool = false,
-    max_inner_iterations = 1000)
+        M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2,
+        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+            abstol = nothing,
+            reltol = nothing),
+        batched::Bool = false,
+        max_inner_iterations = 1000)
     if batched
         return BatchedSimpleDFSane(; σₘᵢₙ = σ_min,
             σₘₐₓ = σ_max,
@@ -98,8 +98,8 @@ function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real =
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
-    args...; abstol = nothing, reltol = nothing, maxiters = 1000,
-    kwargs...)
+        args...; abstol = nothing, reltol = nothing, maxiters = 1000,
+        kwargs...)
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
 
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index de1079beb..eb2ea1f5f 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -4,8 +4,8 @@
 struct Falsi <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
-    maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-    kwargs...)
+        maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
+        kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index 2c0496cfe..8107dde31 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -30,16 +30,16 @@ and static array problems.
 """
 struct SimpleHalley{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     function SimpleHalley(; chunk_size = Val{0}(), autodiff = Val{true}(),
-        diff_type = Val{:forward})
+            diff_type = Val{:forward})
         new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
             SciMLBase._unwrap_val(diff_type)}()
     end
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-    alg::SimpleHalley, args...; abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000, kwargs...)
+        alg::SimpleHalley, args...; abstol = nothing,
+        reltol = nothing,
+        maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fx = f(x)
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index f6688381c..3147c526e 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -59,8 +59,8 @@ struct ITP{T} <: AbstractBracketingAlgorithm
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
-    args...; abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-    maxiters = 1000, kwargs...)
+        args...; abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
+        maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan # a and b
     fl, fr = f(left), f(right)
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 799ba5987..e6a38ecc2 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -9,9 +9,9 @@ This method is non-allocating on scalar problems.
 struct Klement <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-    alg::Klement, args...; abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000, kwargs...)
+        alg::Klement, args...; abstol = nothing,
+        reltol = nothing,
+        maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fₙ = f(x)
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
index fc2b51a88..482092151 100644
--- a/lib/SimpleNonlinearSolve/src/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -18,16 +18,16 @@ struct LBroyden{batched, TC <: NLSolveTerminationCondition} <:
     threshold::Int
 
     function LBroyden(; batched = false, threshold::Int = 27,
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing))
+            termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
+                abstol = nothing,
+                reltol = nothing))
         return new{batched, typeof(termination_condition)}(termination_condition, threshold)
     end
 end
 
 @views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden{batched}, args...;
-    abstol = nothing, reltol = nothing, maxiters = 1000,
-    kwargs...) where {batched}
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        kwargs...) where {batched}
     tc = alg.termination_condition
     mode = DiffEqBase.get_termination_mode(tc)
     threshold = min(maxiters, alg.threshold)
@@ -116,26 +116,26 @@ function _init_lbroyden_state(batched::Bool, x, threshold)
 end
 
 function _rmatvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
-    x::Union{<:AbstractVector, <:Number})
+        x::Union{<:AbstractVector, <:Number})
     length(U) == 0 && return x
     return -x .+ vec((x' * Vᵀ) * U)
 end
 
 function _rmatvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
-    x::AbstractMatrix) where {T1, T2}
+        x::AbstractMatrix) where {T1, T2}
     length(U) == 0 && return x
     Vᵀx = sum(Vᵀ .* reshape(x, size(x, 1), 1, size(x, 2)); dims = 1)
     return -x .+ _drdims_sum(U .* permutedims(Vᵀx, (2, 1, 3)); dims = 1)
 end
 
 function _matvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
-    x::Union{<:AbstractVector, <:Number})
+        x::Union{<:AbstractVector, <:Number})
     length(U) == 0 && return x
     return -x .+ vec(Vᵀ * (U * x))
 end
 
 function _matvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
-    x::AbstractMatrix) where {T1, T2}
+        x::AbstractMatrix) where {T1, T2}
     length(U) == 0 && return x
     xUᵀ = sum(reshape(x, size(x, 1), 1, size(x, 2)) .* permutedims(U, (2, 1, 3)); dims = 1)
     return -x .+ _drdims_sum(xUᵀ .* Vᵀ; dims = 2)
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index d5bad8d13..138e6724d 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -34,10 +34,10 @@ and static array problems.
 struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT} end
 
 function SimpleNewtonRaphson(; batched = false,
-    chunk_size = Val{0}(),
-    autodiff = Val{true}(),
-    diff_type = Val{:forward},
-    termination_condition = missing)
+        chunk_size = Val{0}(),
+        autodiff = Val{true}(),
+        diff_type = Val{:forward},
+        termination_condition = missing)
     if !ismissing(termination_condition) && !batched
         throw(ArgumentError("`termination_condition` is currently only supported for batched problems"))
     end
@@ -63,10 +63,10 @@ end
 
 const SimpleGaussNewton = SimpleNewtonRaphson
 
-function SciMLBase.__solve(prob::Union{NonlinearProblem,NonlinearLeastSquaresProblem},
-    alg::SimpleNewtonRaphson, args...; abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000, kwargs...)
+function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+        alg::SimpleNewtonRaphson, args...; abstol = nothing,
+        reltol = nothing,
+        maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     fx = float(prob.u0)
@@ -76,7 +76,8 @@ function SciMLBase.__solve(prob::Union{NonlinearProblem,NonlinearLeastSquaresPro
         error("SimpleNewtonRaphson currently only supports out-of-place nonlinear problems")
     end
 
-    if prob isa NonlinearLeastSquaresProblem && !(typeof(prob.u0) <: Union{Number, AbstractVector})
+    if prob isa NonlinearLeastSquaresProblem &&
+       !(typeof(prob.u0) <: Union{Number, AbstractVector})
         error("SimpleGaussNewton only supports Number and AbstactVector types. Please convert any problem of AbstractArray into one with u0 as AbstractVector")
     end
 
diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index ce95a178a..eabd7b2ac 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -7,8 +7,8 @@ A non-allocating ridder method
 struct Ridder <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
-    maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-    kwargs...)
+        maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
+        kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index ab28e1ce3..98a36d063 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -64,16 +64,16 @@ struct SimpleTrustRegion{T, CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     expand_factor::T
     max_shrink_times::Int
     function SimpleTrustRegion(; chunk_size = Val{0}(),
-        autodiff = Val{true}(),
-        diff_type = Val{:forward},
-        max_trust_radius::Real = 0.0,
-        initial_trust_radius::Real = 0.0,
-        step_threshold::Real = 0.0001,
-        shrink_threshold::Real = 0.25,
-        expand_threshold::Real = 0.75,
-        shrink_factor::Real = 0.25,
-        expand_factor::Real = 2.0,
-        max_shrink_times::Int = 32)
+            autodiff = Val{true}(),
+            diff_type = Val{:forward},
+            max_trust_radius::Real = 0.0,
+            initial_trust_radius::Real = 0.0,
+            step_threshold::Real = 0.0001,
+            shrink_threshold::Real = 0.25,
+            expand_threshold::Real = 0.75,
+            shrink_factor::Real = 0.25,
+            expand_factor::Real = 2.0,
+            max_shrink_times::Int = 32)
         new{typeof(initial_trust_radius),
             SciMLBase._unwrap_val(chunk_size),
             SciMLBase._unwrap_val(autodiff),
@@ -89,9 +89,9 @@ struct SimpleTrustRegion{T, CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem,
-    alg::SimpleTrustRegion, args...; abstol = nothing,
-    reltol = nothing,
-    maxiters = 1000, kwargs...)
+        alg::SimpleTrustRegion, args...; abstol = nothing,
+        reltol = nothing,
+        maxiters = 1000, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
     T = typeof(x)
@@ -147,7 +147,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
 
         # Compute the ratio of the actual to predicted reduction.
         model = -(δ' * g + 0.5 * δ' * H * δ)
-        r = (fₖ - fₖ₊₁) / model 
+        r = (fₖ - fₖ₊₁) / model
 
         # Update the trust region radius.
         if r < η₂
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 1c4400026..45df96410 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -36,10 +36,10 @@ value_derivative(f::F, x::AbstractArray) where {F} = f(x), ForwardDiff.jacobian(
 Inplace version of [`SimpleNonlinearSolve.value_derivative`](@ref).
 """
 function value_derivative!(J::AbstractMatrix,
-    y::AbstractArray,
-    f!::F,
-    x::AbstractArray,
-    cfg::ForwardDiff.JacobianConfig = ForwardDiff.JacobianConfig(f!, y, x)) where {F}
+        y::AbstractArray,
+        f!::F,
+        x::AbstractArray,
+        cfg::ForwardDiff.JacobianConfig = ForwardDiff.JacobianConfig(f!, y, x)) where {F}
     ForwardDiff.jacobian!(J, f!, y, x, cfg)
     return y, J
 end
diff --git a/lib/SimpleNonlinearSolve/test/inplace.jl b/lib/SimpleNonlinearSolve/test/inplace.jl
index d4882105b..2e9d033a8 100644
--- a/lib/SimpleNonlinearSolve/test/inplace.jl
+++ b/lib/SimpleNonlinearSolve/test/inplace.jl
@@ -4,8 +4,8 @@ using SimpleNonlinearSolve,
 
 # Supported Solvers: BatchedBroyden, BatchedSimpleDFSane, BatchedSimpleNewtonRaphson
 function f!(du::AbstractArray{<:Number, N},
-    u::AbstractArray{<:Number, N},
-    p::AbstractVector) where {N}
+        u::AbstractArray{<:Number, N},
+        p::AbstractVector) where {N}
     u_ = reshape(u, :, size(u, N))
     du .= reshape(sum(abs2, u_; dims = 1) .- u_ .- reshape(p, 1, :), size(u))
     return du

From 000f4db4c8214115bff743a3045cf7a93edfb39a Mon Sep 17 00:00:00 2001
From: FHoltorf <holtorf93@gmail.com>
Date: Tue, 21 Nov 2023 22:44:59 -0500
Subject: [PATCH 216/375] Incorrect RetCode upon termination due to stalling of
 iterates or repeated shrinking of TR

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 98a36d063..72d8efa88 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -155,7 +155,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             shrink_counter += 1
             if shrink_counter > max_shrink_times
                 return SciMLBase.build_solution(prob, alg, x, F;
-                    retcode = ReturnCode.Success)
+                    retcode = ReturnCode.ConvergenceFailure)
             end
         else
             shrink_counter = 0
@@ -163,7 +163,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         if r > η₁
             if isapprox(xₖ₊₁, x, atol = atol, rtol = rtol)
                 return SciMLBase.build_solution(prob, alg, xₖ₊₁, Fₖ₊₁;
-                    retcode = ReturnCode.Success)
+                    retcode = ReturnCode.ConvergenceFailure)
             end
             # Take the step.
             x = xₖ₊₁

From 0cc84ab6f167f967808ec76f67c9c149ead2186c Mon Sep 17 00:00:00 2001
From: FHoltorf <holtorf93@gmail.com>
Date: Wed, 22 Nov 2023 11:51:38 -0500
Subject: [PATCH 217/375] compute Newton step as opposed to Gauss-Newton step
 when J is square and full rank

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 2 +-
 lib/SimpleNonlinearSolve/src/utils.jl       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 72d8efa88..011ececd8 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -140,7 +140,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
 
     for k in 1:maxiters
         # Solve the trust region subproblem.
-        δ = dogleg_method(H, g, Δ)
+        δ = dogleg_method(∇f, F, g, Δ)
         xₖ₊₁ = x + δ
         Fₖ₊₁ = f(xₖ₊₁)
         fₖ₊₁ = 0.5 * norm(Fₖ₊₁)^2
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 45df96410..af66f63f8 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -58,9 +58,9 @@ function init_J(x)
     return J
 end
 
-function dogleg_method(H, g, Δ)
+function dogleg_method(J, f, g, Δ)
     # Compute the Newton step.
-    δN = -H \ g
+    δN = J \ (-f)
     # Test if the full step is within the trust region.
     if norm(δN) ≤ Δ
         return δN

From 8fba768946b1fdf1bae918f9808f131ea1b4ec91 Mon Sep 17 00:00:00 2001
From: FHoltorf <holtorf93@gmail.com>
Date: Wed, 22 Nov 2023 11:54:23 -0500
Subject: [PATCH 218/375] undo ReturnCode changes => treat in another issue

---
 lib/SimpleNonlinearSolve/src/trustRegion.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 011ececd8..0fba7b12f 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -155,7 +155,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
             shrink_counter += 1
             if shrink_counter > max_shrink_times
                 return SciMLBase.build_solution(prob, alg, x, F;
-                    retcode = ReturnCode.ConvergenceFailure)
+                    retcode = ReturnCode.Success)
             end
         else
             shrink_counter = 0
@@ -163,7 +163,7 @@ function SciMLBase.__solve(prob::NonlinearProblem,
         if r > η₁
             if isapprox(xₖ₊₁, x, atol = atol, rtol = rtol)
                 return SciMLBase.build_solution(prob, alg, xₖ₊₁, Fₖ₊₁;
-                    retcode = ReturnCode.ConvergenceFailure)
+                    retcode = ReturnCode.Success)
             end
             # Take the step.
             x = xₖ₊₁

From 19436d1842233a527cc6186ded6530fd22e9c701 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 21 Nov 2023 18:54:23 -0500
Subject: [PATCH 219/375] Start cleaning up simplenonlinearsolve

---
 lib/SimpleNonlinearSolve/.JuliaFormatter.toml |   5 +-
 lib/SimpleNonlinearSolve/Project.toml         |   4 +-
 lib/SimpleNonlinearSolve/README.md            |  17 +-
 .../src/SimpleNonlinearSolve.jl               | 162 ++++----
 lib/SimpleNonlinearSolve/src/alefeld.jl       |   4 +-
 lib/SimpleNonlinearSolve/src/bisection.jl     |   8 +-
 lib/SimpleNonlinearSolve/src/brent.jl         |   1 -
 lib/SimpleNonlinearSolve/src/broyden.jl       |  99 ++---
 lib/SimpleNonlinearSolve/src/dfsane.jl        |  67 ++--
 lib/SimpleNonlinearSolve/src/halley.jl        |  26 +-
 lib/SimpleNonlinearSolve/src/itp.jl           |  24 +-
 lib/SimpleNonlinearSolve/src/raphson.jl       | 122 ++----
 lib/SimpleNonlinearSolve/src/ridder.jl        |   1 -
 lib/SimpleNonlinearSolve/src/utils.jl         | 360 ++++++++++++++----
 14 files changed, 528 insertions(+), 372 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.JuliaFormatter.toml b/lib/SimpleNonlinearSolve/.JuliaFormatter.toml
index 453925c3f..4d06911d7 100644
--- a/lib/SimpleNonlinearSolve/.JuliaFormatter.toml
+++ b/lib/SimpleNonlinearSolve/.JuliaFormatter.toml
@@ -1 +1,4 @@
-style = "sciml"
\ No newline at end of file
+style = "sciml"
+format_markdown = true
+annotate_untyped_fields_with_any = false
+format_docstrings = true
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index aa45b60af..f9242c69f 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,10 +1,12 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.25"
+version = "0.1.26"
 
 [deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
+ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index 53b32e311..efa1fdd63 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -6,12 +6,12 @@
 [![codecov](https://codecov.io/gh/SciML/SimpleNonlinearSolve.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/SciML/SimpleNonlinearSolve.jl)
 [![Build Status](https://github.com/SciML/SimpleNonlinearSolve.jl/workflows/CI/badge.svg)](https://github.com/SciML/SimpleNonlinearSolve.jl/actions?query=workflow%3ACI)
 
-[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac)
+[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor%27s%20Guide-blueviolet)](https://github.com/SciML/ColPrac)
 [![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle)
 
 Fast implementations of root finding algorithms in Julia that satisfy the SciML common interface.
 SimpleNonlinearSolve.jl focuses on low-dependency implementations of very fast methods for
-very small and simple problems. For the full set of solvers, see 
+very small and simple problems. For the full set of solvers, see
 [NonlinearSolve.jl](https://github.com/SciML/NonlinearSolve.jl), of which
 SimpleNonlinearSolve.jl is just one solver set.
 
@@ -25,7 +25,7 @@ the documentation which contains the unreleased features.
 ```julia
 using SimpleNonlinearSolve, StaticArrays
 
-f(u,p) = u .* u .- 2
+f(u, p) = u .* u .- 2
 u0 = @SVector[1.0, 1.0]
 probN = NonlinearProblem{false}(f, u0)
 solver = solve(probN, SimpleNewtonRaphson(), abstol = 1e-9)
@@ -39,3 +39,14 @@ sol = solve(probB, ITP())
 ```
 
 For more details on the bracketing methods, refer to the [Tutorials](https://docs.sciml.ai/NonlinearSolve/stable/tutorials/nonlinear/#Using-Bracketing-Methods) and detailed [APIs](https://docs.sciml.ai/NonlinearSolve/stable/api/simplenonlinearsolve/#Solver-API)
+
+## Breaking Changes in v2
+
+* Batched solvers have been removed in favor of `BatchedArrays.jl`. Stay tuned for detailed
+  tutorials on how to use `BatchedArrays.jl` with `NonlinearSolve` & `SimpleNonlinearSolve`
+  solvers.
+* The old style of specifying autodiff with `chunksize`, `standardtag`, etc. has been
+  deprecated in favor of directly specifying the autodiff type, like `AutoForwardDiff`.
+* `Broyden` and `Klement` have been renamed to `SimpleBroyden` and `SimpleKlement` to
+  avoid conflicts with `NonlinearSolve.jl`'s `GeneralBroyden` and `GeneralKlement`, which
+  will be renamed to `Broyden` and `Klement` in the future.
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 8c84c4377..7d04c1037 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -1,90 +1,96 @@
 module SimpleNonlinearSolve
 
-using Reexport
-using FiniteDiff, ForwardDiff
-using ForwardDiff: Dual
-using StaticArraysCore
-using LinearAlgebra
-import ArrayInterface
-using DiffEqBase
+import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidations
+
+@recompile_invalidations begin
+    using ADTypes,
+        ArrayInterface, ConcreteStructs, DiffEqBase, Reexport, LinearAlgebra,
+        SciMLBase
+
+    import DiffEqBase: AbstractNonlinearTerminationMode,
+        AbstractSafeNonlinearTerminationMode, AbstractSafeBestNonlinearTerminationMode,
+        NonlinearSafeTerminationReturnCode, get_termination_mode
+    using FiniteDiff, ForwardDiff
+    import ForwardDiff: Dual
+    import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace
+    import StaticArraysCore: StaticArray, SVector, SArray, MArray
+end
 
-@reexport using SciMLBase
+@reexport using ADTypes, SciMLBase
 
-const NNlibExtLoaded = Ref{Bool}(false)
+# const NNlibExtLoaded = Ref{Bool}(false)
 
-abstract type AbstractSimpleNonlinearSolveAlgorithm <: SciMLBase.AbstractNonlinearAlgorithm end
+abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorithm end
 abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
-abstract type AbstractNewtonAlgorithm{CS, AD, FDT} <: AbstractSimpleNonlinearSolveAlgorithm end
-abstract type AbstractImmutableNonlinearSolver <: AbstractSimpleNonlinearSolveAlgorithm end
-abstract type AbstractBatchedNonlinearSolveAlgorithm <:
-              AbstractSimpleNonlinearSolveAlgorithm end
+abstract type AbstractNewtonAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 
 include("utils.jl")
-include("bisection.jl")
-include("falsi.jl")
+# include("bisection.jl")
+# include("falsi.jl")
 include("raphson.jl")
 include("broyden.jl")
-include("lbroyden.jl")
-include("klement.jl")
-include("trustRegion.jl")
-include("ridder.jl")
-include("brent.jl")
-include("dfsane.jl")
-include("ad.jl")
-include("halley.jl")
-include("alefeld.jl")
-include("itp.jl")
-
-# Batched Solver Support
-include("batched/utils.jl")
-include("batched/raphson.jl")
-include("batched/dfsane.jl")
-include("batched/broyden.jl")
-
-## Default algorithm
-
-# Set the default bracketing method to ITP
-
-function SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...)
-    SciMLBase.solve(prob, ITP(); kwargs...)
-end
-
-function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing,
-        args...; kwargs...)
-    SciMLBase.solve(prob, ITP(), args...; kwargs...)
-end
-
-import PrecompileTools
-
-PrecompileTools.@compile_workload begin
-    for T in (Float32, Float64)
-        prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-        for alg in (SimpleNewtonRaphson, SimpleHalley, Broyden, Klement, SimpleTrustRegion,
-            SimpleDFSane)
-            solve(prob_no_brack, alg(), abstol = T(1e-2))
-        end
-
-        #=
-        for alg in (SimpleNewtonRaphson,)
-            for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
-                u0 = T.(.1)
-                probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
-                solve(probN, alg(), tol = T(1e-2))
-            end
-        end
-        =#
-
-        prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
-            T.((0.0, 2.0)),
-            T(2))
-        for alg in (Bisection, Falsi, Ridder, Brent, Alefeld, ITP)
-            solve(prob_brack, alg(), abstol = T(1e-2))
-        end
-    end
-end
-
-export Bisection, Brent, Broyden, LBroyden, SimpleDFSane, Falsi, SimpleHalley, Klement,
-    Ridder, SimpleNewtonRaphson, SimpleTrustRegion, Alefeld, ITP, SimpleGaussNewton
-export BatchedBroyden, BatchedSimpleNewtonRaphson, BatchedSimpleDFSane
+# include("lbroyden.jl")
+# include("klement.jl")
+# include("trustRegion.jl")
+# include("ridder.jl")
+# include("brent.jl")
+# include("dfsane.jl")
+# include("ad.jl")
+# include("halley.jl")
+# include("alefeld.jl")
+# include("itp.jl")
+
+# # Batched Solver Support
+# include("batched/utils.jl")
+# include("batched/raphson.jl")
+# include("batched/dfsane.jl")
+# include("batched/broyden.jl")
+
+# ## Default algorithm
+
+# # Set the default bracketing method to ITP
+
+# function SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...)
+#     SciMLBase.solve(prob, ITP(); kwargs...)
+# end
+
+# function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing,
+#     args...; kwargs...)
+#     SciMLBase.solve(prob, ITP(), args...; kwargs...)
+# end
+
+# import PrecompileTools
+
+# PrecompileTools.@compile_workload begin
+#     for T in (Float32, Float64)
+#         prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
+#         for alg in (SimpleNewtonRaphson, SimpleHalley, Broyden, Klement, SimpleTrustRegion,
+#             SimpleDFSane)
+#             solve(prob_no_brack, alg(), abstol = T(1e-2))
+#         end
+
+#         #=
+#         for alg in (SimpleNewtonRaphson,)
+#             for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
+#                 u0 = T.(.1)
+#                 probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
+#                 solve(probN, alg(), tol = T(1e-2))
+#             end
+#         end
+#         =#
+
+#         prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
+#             T.((0.0, 2.0)),
+#             T(2))
+#         for alg in (Bisection, Falsi, Ridder, Brent, Alefeld, ITP)
+#             solve(prob_brack, alg(), abstol = T(1e-2))
+#         end
+#     end
+# end
+
+export SimpleBroyden, SimpleGaussNewton, SimpleNewtonRaphson
+# export Bisection, Brent, LBroyden, SimpleDFSane, Falsi, SimpleHalley, Klement,
+#     Ridder, SimpleTrustRegion, Alefeld, ITP
+# export BatchedBroyden, BatchedSimpleDFSane
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 0d4f56116..3d3b2ada8 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -1,9 +1,9 @@
 """
-`Alefeld()` 
+`Alefeld()`
 
 An implementation of algorithm 4.2 from [Alefeld](https://dl.acm.org/doi/10.1145/210089.210111).
 
-The paper brought up two new algorithms. Here choose to implement algorithm 4.2 rather than 
+The paper brought up two new algorithms. Here choose to implement algorithm 4.2 rather than
 algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal procedure.
 """
 struct Alefeld <: AbstractBracketingAlgorithm end
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index f7c98aa65..93b1cbeb0 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -5,10 +5,10 @@ A common bisection method.
 
 ### Keyword Arguments
 
-- `exact_left`: whether to enforce whether the left side of the interval must be exactly
-  zero for the returned result. Defaults to false.
-- `exact_right`: whether to enforce whether the right side of the interval must be exactly
-  zero for the returned result. Defaults to false.
+  - `exact_left`: whether to enforce whether the left side of the interval must be exactly
+    zero for the returned result. Defaults to false.
+  - `exact_right`: whether to enforce whether the right side of the interval must be exactly
+    zero for the returned result. Defaults to false.
 """
 struct Bisection <: AbstractBracketingAlgorithm
     exact_left::Bool
diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index 7d7a6bcf9..1319ed979 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -2,7 +2,6 @@
 `Brent()`
 
 A non-allocating Brent method
-
 """
 struct Brent <: AbstractBracketingAlgorithm end
 
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 07b2609f9..4b7d5d902 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -1,79 +1,52 @@
 """
-    Broyden(; batched = false,
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing))
+    SimpleBroyden()
 
 A low-overhead implementation of Broyden. This method is non-allocating on scalar
 and static array problems.
-
-!!! note
-
-    To use the `batched` version, remember to load `NNlib`, i.e., `using NNlib` or
-    `import NNlib` must be present in your code.
 """
-struct Broyden{TC <: NLSolveTerminationCondition} <:
-       AbstractSimpleNonlinearSolveAlgorithm
-    termination_condition::TC
-end
+struct SimpleBroyden <: AbstractSimpleNonlinearSolveAlgorithm end
 
-function Broyden(; batched = false,
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing))
-    if batched
-        @assert NNlibExtLoaded[] "Please install and load `NNlib.jl` to use batched Broyden."
-        return BatchedBroyden(termination_condition)
-    end
-    return Broyden(termination_condition)
-end
-
-function SciMLBase.__solve(prob::NonlinearProblem, alg::Broyden, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
-    tc = alg.termination_condition
-    mode = DiffEqBase.get_termination_mode(tc)
-    f = Base.Fix2(prob.f, prob.p)
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        termination_condition = nothing, kwargs...)
+    f = isinplace(prob) ? (du, u) -> prob.f(du, u, prob.p) : u -> prob.f(u, prob.p)
     x = float(prob.u0)
+    fx = _get_fx(prob, x)
+    xo, δx, fprev, δf = __copy(x), __copy(x), __copy(fx), __copy(fx)
 
-    fₙ = f(x)
-    T = eltype(x)
-    J⁻¹ = init_J(x)
+    J⁻¹ = __init_identity_jacobian(fx, x)
+    J⁻¹δf, xᵀJ⁻¹ = __copy(x), __copy(x)
+    δJ⁻¹, δJ⁻¹n = __copy(x, J⁻¹), __copy(x)
 
-    if SciMLBase.isinplace(prob)
-        error("Broyden currently only supports out-of-place nonlinear problems")
-    end
+    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+        termination_condition)
 
-    atol = _get_tolerance(abstol, tc.abstol, T)
-    rtol = _get_tolerance(reltol, tc.reltol, T)
-
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        error("Broyden currently doesn't support SAFE_BEST termination modes")
-    end
-
-    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
-              nothing
-    termination_condition = tc(storage)
-
-    xₙ = x
-    xₙ₋₁ = x
-    fₙ₋₁ = fₙ
     for _ in 1:maxiters
-        xₙ = xₙ₋₁ - _restructure(xₙ₋₁, J⁻¹ * _vec(fₙ₋₁))
-        fₙ = f(xₙ)
-        Δxₙ = xₙ - xₙ₋₁
-        Δfₙ = fₙ - fₙ₋₁
-        J⁻¹Δfₙ = _restructure(Δfₙ, J⁻¹ * _vec(Δfₙ))
-        J⁻¹ += _restructure(J⁻¹,
-            ((_vec(Δxₙ) .- _vec(J⁻¹Δfₙ)) ./ (_vec(Δxₙ)' * _vec(J⁻¹Δfₙ))) *
-            (_vec(Δxₙ)' * J⁻¹))
-
-        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
+        δx = _restructure(δx, __mul!!(_vec(δx), J⁻¹, _vec(fprev)))
+        x = __sub!!(x, xo, δx)
+        fx = __eval_f(prob, f, fx, x)
+        δf = __sub!!(δf, fx, fprev)
+
+        # Termination Checks
+        tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+        tc_sol !== nothing && return tc_sol
+
+        J⁻¹δf = _restructure(J⁻¹δf, __mul!!(_vec(J⁻¹δf), J⁻¹, _vec(δf)))
+        d = dot(δx, J⁻¹δf)
+        xᵀJ⁻¹ = _restructure(xᵀJ⁻¹, __mul!!(_vec(xᵀJ⁻¹), _vec(δx)', J⁻¹))
+
+        if ArrayInterface.can_setindex(δJ⁻¹n)
+            @. δJ⁻¹n = (δx - J⁻¹δf) / d
+        else
+            δJ⁻¹n = @. (δx - J⁻¹δf) / d
         end
 
-        xₙ₋₁ = xₙ
-        fₙ₋₁ = fₙ
+        δJ⁻¹ = __mul!!(δJ⁻¹, δJ⁻¹n, xᵀJ⁻¹')
+        J⁻¹ = __add!!(J⁻¹, δJ⁻¹)
+
+        xo = __copyto!!(xo, x)
+        fprev = __copyto!!(fprev, fx)
     end
 
-    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+    return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index 49c50bca3..e7fda8629 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -16,40 +16,39 @@ Computation, 75, 1429-1448.](https://www.researchgate.net/publication/220576479_
 
 ### Keyword Arguments
 
-- `σ_min`: the minimum value of the spectral coefficient `σ_k` which is related to the step
-  size in the algorithm. Defaults to `1e-10`.
-- `σ_max`: the maximum value of the spectral coefficient `σ_k` which is related to the step
-  size in the algorithm. Defaults to `1e10`.
-- `σ_1`: the initial value of the spectral coefficient `σ_k` which is related to the step
-  size in the algorithm.. Defaults to `1.0`.
-- `M`: The monotonicity of the algorithm is determined by a this positive integer.
-  A value of 1 for `M` would result in strict monotonicity in the decrease of the L2-norm
-  of the function `f`. However, higher values allow for more flexibility in this reduction.
-  Despite this, the algorithm still ensures global convergence through the use of a
-  non-monotone line-search algorithm that adheres to the Grippo-Lampariello-Lucidi
-  condition. Values in the range of 5 to 20 are usually sufficient, but some cases may call
-  for a higher value of `M`. The default setting is 10.
-- `γ`: a parameter that influences if a proposed step will be accepted. Higher value of `γ`
-  will make the algorithm more restrictive in accepting steps. Defaults to `1e-4`.
-- `τ_min`: if a step is rejected the new step size will get multiplied by factor, and this
-  parameter is the minimum value of that factor. Defaults to `0.1`.
-- `τ_max`: if a step is rejected the new step size will get multiplied by factor, and this
-  parameter is the maximum value of that factor. Defaults to `0.5`.
-- `nexp`: the exponent of the loss, i.e. ``f_k=||F(x_k)||^{nexp}``. The paper uses
-  `nexp ∈ {1,2}`. Defaults to `2`.
-- `η_strategy`:  function to determine the parameter `η_k`, which enables growth
-  of ``||F||^2``. Called as ``η_k = η_strategy(f_1, k, x, F)`` with `f_1` initialized as
-  ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and
-  `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to
-  ``||F||^2 / k^2``.
-- `termination_condition`: a `NLSolveTerminationCondition` that determines when the solver
-  should terminate. Defaults to `NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-  abstol = nothing, reltol = nothing)`.
-- `batched`: if `true`, the algorithm will use a batched version of the algorithm that treats each
-  column of `x` as a separate problem. This can be useful nonlinear problems involing neural
-  networks. Defaults to `false`.
-- `max_inner_iterations`: the maximum number of iterations allowed for the inner loop of the
-  algorithm. Used exclusively in `batched` mode. Defaults to `1000`.
+  - `σ_min`: the minimum value of the spectral coefficient `σ_k` which is related to the step
+    size in the algorithm. Defaults to `1e-10`.
+  - `σ_max`: the maximum value of the spectral coefficient `σ_k` which is related to the step
+    size in the algorithm. Defaults to `1e10`.
+  - `σ_1`: the initial value of the spectral coefficient `σ_k` which is related to the step
+    size in the algorithm.. Defaults to `1.0`.
+  - `M`: The monotonicity of the algorithm is determined by a this positive integer.
+    A value of 1 for `M` would result in strict monotonicity in the decrease of the L2-norm
+    of the function `f`. However, higher values allow for more flexibility in this reduction.
+    Despite this, the algorithm still ensures global convergence through the use of a
+    non-monotone line-search algorithm that adheres to the Grippo-Lampariello-Lucidi
+    condition. Values in the range of 5 to 20 are usually sufficient, but some cases may call
+    for a higher value of `M`. The default setting is 10.
+  - `γ`: a parameter that influences if a proposed step will be accepted. Higher value of `γ`
+    will make the algorithm more restrictive in accepting steps. Defaults to `1e-4`.
+  - `τ_min`: if a step is rejected the new step size will get multiplied by factor, and this
+    parameter is the minimum value of that factor. Defaults to `0.1`.
+  - `τ_max`: if a step is rejected the new step size will get multiplied by factor, and this
+    parameter is the maximum value of that factor. Defaults to `0.5`.
+  - `nexp`: the exponent of the loss, i.e. ``f_k=||F(x_k)||^{nexp}``. The paper uses
+    `nexp ∈ {1,2}`. Defaults to `2`.
+  - `η_strategy`:  function to determine the parameter `η_k`, which enables growth
+    of ``||F||^2``. Called as ``η_k = η_strategy(f_1, k, x, F)`` with `f_1` initialized as
+    ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and
+    `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to
+    ``||F||^2 / k^2``.
+  - `termination_condition`: a `NLSolveTerminationCondition` that determines when the solver
+    should terminate. Defaults to `NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault; abstol = nothing, reltol = nothing)`.
+  - `batched`: if `true`, the algorithm will use a batched version of the algorithm that treats each
+    column of `x` as a separate problem. This can be useful nonlinear problems involing neural
+    networks. Defaults to `false`.
+  - `max_inner_iterations`: the maximum number of iterations allowed for the inner loop of the
+    algorithm. Used exclusively in `batched` mode. Defaults to `1000`.
 """
 struct SimpleDFSane{T, TC} <: AbstractSimpleNonlinearSolveAlgorithm
     σ_min::T
diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
index 8107dde31..8131acada 100644
--- a/lib/SimpleNonlinearSolve/src/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -1,7 +1,7 @@
 """
 ```julia
 SimpleHalley(; chunk_size = Val{0}(), autodiff = Val{true}(),
-                                 diff_type = Val{:forward})
+    diff_type = Val{:forward})
 ```
 
 A low-overhead implementation of SimpleHalley's Method. This method is non-allocating on scalar
@@ -15,18 +15,18 @@ and static array problems.
 
 ### Keyword Arguments
 
-- `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
-  system. This allows for multiple derivative columns to be computed simultaneously,
-  improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
-  default chunk size mechanism. For more details, see the documentation for
-  [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
-- `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
-  Note that this argument is ignored if an analytical Jacobian is passed; as that will be
-  used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
-  If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
-- `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
-  `Val{:forward}` for forward finite differences. For more details on the choices, see the
-  [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+  - `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
+    system. This allows for multiple derivative columns to be computed simultaneously,
+    improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
+    default chunk size mechanism. For more details, see the documentation for
+    [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
+  - `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
+    Note that this argument is ignored if an analytical Jacobian is passed; as that will be
+    used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
+    If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
+  - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
+    `Val{:forward}` for forward finite differences. For more details on the choices, see the
+    [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
 """
 struct SimpleHalley{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     function SimpleHalley(; chunk_size = Val{0}(), autodiff = Val{true}(),
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 3147c526e..933995cec 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -16,18 +16,18 @@ Average Performance Preserving Minmax Optimality"
 
 The following keyword parameters are accepted.
 
-- `n₀::Int = 1`, the 'slack'. Must not be negative.\n
-  When n₀ = 0 the worst-case is identical to that of bisection,
-  but increacing n₀ provides greater oppotunity for superlinearity.
-- `κ₁::Float64 = 0.1`. Must not be negative.\n
-  The recomended value is `0.2/(x₂ - x₁)`.
-  Lower values produce tighter asymptotic behaviour, while higher values
-  improve the steady-state behaviour when truncation is not helpful.
-- `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62).\n
-  Higher values allow for a greater convergence rate,
-  but also make the method more succeptable to worst-case performance.
-  In practice, κ=1,2 seems to work well due to the computational simplicity,
-  as κ₂ is used as an exponent in the method.
+  - `n₀::Int = 1`, the 'slack'. Must not be negative.\n
+    When n₀ = 0 the worst-case is identical to that of bisection,
+    but increacing n₀ provides greater oppotunity for superlinearity.
+  - `κ₁::Float64 = 0.1`. Must not be negative.\n
+    The recomended value is `0.2/(x₂ - x₁)`.
+    Lower values produce tighter asymptotic behaviour, while higher values
+    improve the steady-state behaviour when truncation is not helpful.
+  - `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62).\n
+    Higher values allow for a greater convergence rate,
+    but also make the method more succeptable to worst-case performance.
+    In practice, κ=1,2 seems to work well due to the computational simplicity,
+    as κ₂ is used as an exponent in the method.
 
 ### Worst Case Performance
 
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index 138e6724d..a1974ba88 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -1,9 +1,6 @@
 """
-    SimpleNewtonRaphson(; batched = false,
-        chunk_size = Val{0}(),
-        autodiff = Val{true}(),
-        diff_type = Val{:forward},
-        termination_condition = missing)
+    SimpleNewtonRaphson(autodiff)
+    SimpleNewtonRaphson(; autodiff = AutoForwardDiff())
 
 A low-overhead implementation of Newton-Raphson. This method is non-allocating on scalar
 and static array problems.
@@ -16,110 +13,45 @@ and static array problems.
 
 ### Keyword Arguments
 
-- `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
-  system. This allows for multiple derivative columns to be computed simultaneously,
-  improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
-  default chunk size mechanism. For more details, see the documentation for
-  [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
-- `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
-  Note that this argument is ignored if an analytical Jacobian is passed; as that will be
-  used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
-  If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
-- `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
-  `Val{:forward}` for forward finite differences. For more details on the choices, see the
-  [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
-- `termination_condition`: control the termination of the algorithm. (Only works for batched
-  problems)
+  - `autodiff`: determines the backend used for the Jacobian. Defaults to
+    `AutoForwardDiff()`. Valid choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
 """
-struct SimpleNewtonRaphson{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT} end
-
-function SimpleNewtonRaphson(; batched = false,
-        chunk_size = Val{0}(),
-        autodiff = Val{true}(),
-        diff_type = Val{:forward},
-        termination_condition = missing)
-    if !ismissing(termination_condition) && !batched
-        throw(ArgumentError("`termination_condition` is currently only supported for batched problems"))
-    end
-    if batched
-        # @assert ADLinearSolveFDExtLoaded[] "Please install and load `LinearSolve.jl`, `FiniteDifferences.jl` and `AbstractDifferentiation.jl` to use batched Newton-Raphson."
-        termination_condition = ismissing(termination_condition) ?
-                                NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing) :
-                                termination_condition
-        return BatchedSimpleNewtonRaphson(; chunk_size,
-            autodiff,
-            diff_type,
-            termination_condition)
-        return SimpleNewtonRaphson{SciMLBase._unwrap_val(chunk_size),
-            SciMLBase._unwrap_val(autodiff),
-            SciMLBase._unwrap_val(diff_type)}()
-    end
-    return SimpleNewtonRaphson{SciMLBase._unwrap_val(chunk_size),
-        SciMLBase._unwrap_val(autodiff),
-        SciMLBase._unwrap_val(diff_type)}()
+@concrete struct SimpleNewtonRaphson <: AbstractNewtonAlgorithm
+    ad
 end
 
+SimpleNewtonRaphson(; autodiff = AutoForwardDiff()) = SimpleNewtonRaphson(autodiff)
+
 const SimpleGaussNewton = SimpleNewtonRaphson
 
 function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
-        alg::SimpleNewtonRaphson, args...; abstol = nothing,
-        reltol = nothing,
-        maxiters = 1000, kwargs...)
-    f = Base.Fix2(prob.f, prob.p)
+        alg::SimpleNewtonRaphson, args...; abstol = nothing, reltol = nothing,
+        maxiters = 1000, termination_condition = nothing, kwargs...)
     x = float(prob.u0)
-    fx = float(prob.u0)
-    T = typeof(x)
+    fx = _get_fx(prob, x)
+    xo = __copy(x)
+    J, jac_cache = jacobian_cache(alg.ad, prob.f, fx, x, prob.p)
 
-    if SciMLBase.isinplace(prob)
-        error("SimpleNewtonRaphson currently only supports out-of-place nonlinear problems")
-    end
-
-    if prob isa NonlinearLeastSquaresProblem &&
-       !(typeof(prob.u0) <: Union{Number, AbstractVector})
-        error("SimpleGaussNewton only supports Number and AbstactVector types. Please convert any problem of AbstractArray into one with u0 as AbstractVector")
-    end
-
-    atol = abstol !== nothing ? abstol :
-           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
-    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
-
-    if x isa Number
-        xo = oftype(one(eltype(x)), Inf)
-    else
-        xo = map(x -> oftype(one(eltype(x)), Inf), x)
-    end
+    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+        termination_condition)
 
     for i in 1:maxiters
-        if DiffEqBase.has_jac(prob.f)
-            dfx = prob.f.jac(x, prob.p)
-            fx = f(x)
-        elseif alg_autodiff(alg)
-            fx, dfx = value_derivative(f, x)
-        elseif x isa AbstractArray
-            fx = f(x)
-            dfx = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x), fx)
-        else
-            fx = f(x)
-            dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x),
-                fx)
-        end
-        iszero(fx) &&
-            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+        fx, dfx = value_and_jacobian(alg.ad, prob.f, fx, x, prob.p, jac_cache; J)
 
-        if prob isa NonlinearProblem
-            Δx = _restructure(fx, dfx \ _vec(fx))
+        if i == 1
+            if iszero(fx)
+                return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+            end
         else
-            Δx = dfx \ fx
+            # Termination Checks
+            tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+            tc_sol !== nothing && return tc_sol
         end
 
-        x -= Δx
-        if isapprox(x, xo, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-        end
-        xo = x
+        xo = __copyto!!(xo, x)
+        Δx = _restructure(x, dfx \ _vec(fx))
+        x = __sub!!(x, Δx)
     end
 
-    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+    return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index eabd7b2ac..41b43200e 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -2,7 +2,6 @@
 `Ridder()`
 
 A non-allocating ridder method
-
 """
 struct Ridder <: AbstractBracketingAlgorithm end
 
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index af66f63f8..6a35aae25 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -1,91 +1,323 @@
-"""
-  prevfloat_tdir(x, x0, x1)
+struct SimpleNonlinearSolveTag end
 
-Move `x` one floating point towards x0.
-"""
-function prevfloat_tdir(x, x0, x1)
-    x1 > x0 ? prevfloat(x) : nextfloat(x)
+function ForwardDiff.checktag(::Type{<:ForwardDiff.Tag{<:SimpleNonlinearSolveTag, <:T}},
+        f::F, x::AbstractArray{T}) where {T, F}
+    return true
 end
 
-function nextfloat_tdir(x, x0, x1)
-    x1 > x0 ? nextfloat(x) : prevfloat(x)
-end
+# """
+#   prevfloat_tdir(x, x0, x1)
 
-function max_tdir(a, b, x0, x1)
-    x1 > x0 ? max(a, b) : min(a, b)
-end
+# Move `x` one floating point towards x0.
+# """
+# function prevfloat_tdir(x, x0, x1)
+#     x1 > x0 ? prevfloat(x) : nextfloat(x)
+# end
 
-alg_autodiff(alg::AbstractNewtonAlgorithm{CS, AD, FDT}) where {CS, AD, FDT} = AD
-diff_type(alg::AbstractNewtonAlgorithm{CS, AD, FDT}) where {CS, AD, FDT} = FDT
+# function nextfloat_tdir(x, x0, x1)
+#     x1 > x0 ? nextfloat(x) : prevfloat(x)
+# end
 
-"""
-  value_derivative(f, x)
+# function max_tdir(a, b, x0, x1)
+#     x1 > x0 ? max(a, b) : min(a, b)
+# end
 
-Compute `f(x), d/dx f(x)` in the most efficient way.
-"""
-function value_derivative(f::F, x::R) where {F, R}
-    T = typeof(ForwardDiff.Tag(f, R))
-    out = f(ForwardDiff.Dual{T}(x, one(x)))
-    ForwardDiff.value(out), ForwardDiff.extract_derivative(T, out)
+# alg_autodiff(alg::AbstractNewtonAlgorithm{CS, AD, FDT}) where {CS, AD, FDT} = AD
+# diff_type(alg::AbstractNewtonAlgorithm{CS, AD, FDT}) where {CS, AD, FDT} = FDT
+
+__standard_tag(::Nothing, x) = ForwardDiff.Tag(SimpleNonlinearSolveTag(), eltype(x))
+__standard_tag(tag::ForwardDiff.Tag, _) = tag
+__standard_tag(tag, x) = ForwardDiff.Tag(tag, eltype(x))
+
+function __get_jacobian_config(ad::AutoForwardDiff{CS}, f, x) where {CS}
+    ck = (CS === nothing || CS ≤ 0) ? ForwardDiff.Chunk(length(x)) : ForwardDiff.Chunk{CS}()
+    tag = __standard_tag(ad.tag, x)
+    return ForwardDiff.JacobianConfig(f, x, ck, tag)
+end
+function __get_jacobian_config(ad::AutoForwardDiff{CS}, f!, y, x) where {CS}
+    ck = (CS === nothing || CS ≤ 0) ? ForwardDiff.Chunk(length(x)) : ForwardDiff.Chunk{CS}()
+    tag = __standard_tag(ad.tag, x)
+    return ForwardDiff.JacobianConfig(f!, y, x, ck, tag)
 end
-value_derivative(f::F, x::AbstractArray) where {F} = f(x), ForwardDiff.jacobian(f, x)
 
 """
-    value_derivative!(J, y, f!, x, cfg = JacobianConfig(f!, y, x))
+    value_and_jacobian(ad, f, y, x, p, cache; J = nothing)
 
-Inplace version of [`SimpleNonlinearSolve.value_derivative`](@ref).
+Compute `f(x), d/dx f(x)` in the most efficient way based on `ad`. None of the arguments
+except `cache` (& `J` if not nothing) are mutated.
 """
-function value_derivative!(J::AbstractMatrix,
-        y::AbstractArray,
-        f!::F,
-        x::AbstractArray,
-        cfg::ForwardDiff.JacobianConfig = ForwardDiff.JacobianConfig(f!, y, x)) where {F}
-    ForwardDiff.jacobian!(J, f!, y, x, cfg)
-    return y, J
-end
-
-value(x) = x
-value(x::Dual) = ForwardDiff.value(x)
-value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
-
-function init_J(x)
-    J = ArrayInterface.zeromatrix(x)
-    if ismutable(x)
-        J[diagind(J)] .= one(eltype(x))
+function value_and_jacobian(ad, f::F, y, x::X, p, cache; J = nothing) where {F, X}
+    if isinplace(f)
+        _f = (du, u) -> f(du, u, p)
+        if DiffEqBase.has_jac(f)
+            f.jac(J, x, p)
+            _f(y, x)
+            return y, J
+        elseif ad isa AutoForwardDiff
+            res = DiffResults.DiffResult(y, J)
+            ForwardDiff.jacobian!(res, _f, y, x, cache)
+            return DiffResults.value(res), DiffResults.jacobian(res)
+        elseif ad isa AutoFiniteDiff
+            FiniteDiff.finite_difference_jacobian!(J, _f, x, cache)
+            _f(y, x)
+            return y, J
+        else
+            throw(ArgumentError("Unsupported AD method: $(ad)"))
+        end
     else
-        J += I
+        _f = Base.Fix2(f, p)
+        if DiffEqBase.has_jac(f)
+            return _f(x), f.jac(x, p)
+        elseif ad isa AutoForwardDiff
+            if ArrayInterface.can_setindex(x)
+                res = DiffResults.DiffResult(y, J)
+                ForwardDiff.jacobian!(res, _f, x, cache)
+                return DiffResults.value(res), DiffResults.jacobian(res)
+            else
+                J_fd = ForwardDiff.jacobian(_f, x, cache)
+                return _f(x), J_fd
+            end
+        elseif ad isa AutoFiniteDiff
+            J_fd = FiniteDiff.finite_difference_jacobian(_f, x, cache)
+            return _f(x), J_fd
+        else
+            throw(ArgumentError("Unsupported AD method: $(ad)"))
+        end
     end
-    return J
 end
 
-function dogleg_method(J, f, g, Δ)
-    # Compute the Newton step.
-    δN = J \ (-f)
-    # Test if the full step is within the trust region.
-    if norm(δN) ≤ Δ
-        return δN
+function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
+    if isinplace(f)
+        _f = (du, u) -> f(du, u, p)
+        J = similar(y, length(y), length(x))
+        if DiffEqBase.has_jac(f)
+            return J, nothing
+        elseif ad isa AutoForwardDiff
+            return J, __get_jacobian_config(ad, _f, y, x)
+        elseif ad isa AutoFiniteDiff
+            return J, FiniteDiff.JacobianCache(copy(x), copy(y), copy(y), ad.fdtype)
+        else
+            throw(ArgumentError("Unsupported AD method: $(ad)"))
+        end
+    else
+        _f = Base.Fix2(f, p)
+        if DiffEqBase.has_jac(f)
+            return nothing, nothing
+        elseif ad isa AutoForwardDiff
+            J = ArrayInterface.can_setindex(x) ? similar(y, length(fx), length(x)) : nothing
+            return J, __get_jacobian_config(ad, _f, x)
+        elseif ad isa AutoFiniteDiff
+            return nothing, FiniteDiff.JacobianCache(copy(x), copy(y), copy(y), ad.fdtype)
+        else
+            throw(ArgumentError("Unsupported AD method: $(ad)"))
+        end
     end
+end
 
-    # Calcualte Cauchy point, optimum along the steepest descent direction.
-    δsd = -g
-    norm_δsd = norm(δsd)
-    if norm_δsd ≥ Δ
-        return δsd .* Δ / norm_δsd
-    end
+# """
+#   value_derivative(f, x)
 
-    # Find the intersection point on the boundary.
-    δN_δsd = δN - δsd
-    dot_δN_δsd = dot(δN_δsd, δN_δsd)
-    dot_δsd_δN_δsd = dot(δsd, δN_δsd)
-    dot_δsd = dot(δsd, δsd)
-    fact = dot_δsd_δN_δsd^2 - dot_δN_δsd * (dot_δsd - Δ^2)
-    tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd
-    return δsd + tau * δN_δsd
+# Compute `f(x), d/dx f(x)` in the most efficient way.
+# """
+# function value_derivative(f::F, x::R) where {F, R}
+#     T = typeof(ForwardDiff.Tag(f, R))
+#     out = f(ForwardDiff.Dual{T}(x, one(x)))
+#     ForwardDiff.value(out), ForwardDiff.extract_derivative(T, out)
+# end
+# value_derivative(f::F, x::AbstractArray) where {F} = f(x), ForwardDiff.jacobian(f, x)
+
+# """
+#     value_derivative!(J, y, f!, x, cfg = JacobianConfig(f!, y, x))
+
+# Inplace version of [`SimpleNonlinearSolve.value_derivative`](@ref).
+# """
+# function value_derivative!(J::AbstractMatrix,
+#     y::AbstractArray,
+#     f!::F,
+#     x::AbstractArray,
+#     cfg::ForwardDiff.JacobianConfig = ForwardDiff.JacobianConfig(f!, y, x)) where {F}
+#     ForwardDiff.jacobian!(J, f!, y, x, cfg)
+#     return y, J
+# end
+
+# value(x) = x
+# value(x::Dual) = ForwardDiff.value(x)
+# value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
+
+__init_identity_jacobian(u::Number, _) = u
+function __init_identity_jacobian(u, fu)
+    J = similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u))
+    J[diagind(J)] .= one(eltype(J))
+    return J
+end
+function __init_identity_jacobian(u::StaticArray, fu)
+    return convert(MArray{Tuple{length(fu), length(u)}},
+        Matrix{eltype(u)}(I, length(fu), length(u)))
 end
 
+# function dogleg_method(J, f, g, Δ)
+#     # Compute the Newton step.
+#     δN = J \ (-f)
+#     # Test if the full step is within the trust region.
+#     if norm(δN) ≤ Δ
+#         return δN
+#     end
+
+#     # Calcualte Cauchy point, optimum along the steepest descent direction.
+#     δsd = -g
+#     norm_δsd = norm(δsd)
+#     if norm_δsd ≥ Δ
+#         return δsd .* Δ / norm_δsd
+#     end
+
+#     # Find the intersection point on the boundary.
+#     δN_δsd = δN - δsd
+#     dot_δN_δsd = dot(δN_δsd, δN_δsd)
+#     dot_δsd_δN_δsd = dot(δsd, δN_δsd)
+#     dot_δsd = dot(δsd, δsd)
+#     fact = dot_δsd_δN_δsd^2 - dot_δN_δsd * (dot_δsd - Δ^2)
+#     tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd
+#     return δsd + tau * δN_δsd
+# end
+
 @inline _vec(v) = vec(v)
 @inline _vec(v::Number) = v
 @inline _vec(v::AbstractVector) = v
 
 @inline _restructure(y::Number, x::Number) = x
 @inline _restructure(y, x) = ArrayInterface.restructure(y, x)
+
+@inline function _get_fx(prob::NonlinearLeastSquaresProblem, x)
+    isinplace(prob) && prob.f.resid_prototype === nothing &&
+        error("Inplace NonlinearLeastSquaresProblem requires a `resid_prototype`")
+    return _get_fx(prob.f, x, prob.p)
+end
+@inline _get_fx(prob::NonlinearProblem, x) = _get_fx(prob.f, x, prob.p)
+@inline function _get_fx(f::NonlinearFunction, x, p)
+    if isinplace(f)
+        if f.resid_prototype !== nothing
+            T = eltype(x)
+            return T.(f.resid_prototype)
+        else
+            fx = similar(x)
+            f(fx, x, p)
+            return fx
+        end
+    else
+        return f(x, p)
+    end
+end
+
+# Termination Conditions Support
+# Taken directly from NonlinearSolve.jl
+function init_termination_cache(abstol, reltol, du, u, ::Nothing)
+    return init_termination_cache(abstol, reltol, du, u, AbsSafeBestTerminationMode())
+end
+function init_termination_cache(abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode)
+    tc_cache = init(du, u, tc; abstol, reltol)
+    return DiffEqBase.get_abstol(tc_cache), DiffEqBase.get_reltol(tc_cache), tc_cache
+end
+
+function check_termination(tc_cache, fx, x, xo, prob, alg)
+    return check_termination(tc_cache, fx, x, xo, prob, alg,
+        DiffEqBase.get_termination_mode(tc_cache))
+end
+function check_termination(tc_cache, fx, x, xo, prob, alg,
+        ::AbstractNonlinearTerminationMode)
+    if tc_cache(fx, x, xo)
+        return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+    end
+    return nothing
+end
+function check_termination(tc_cache, fx, x, xo, prob, alg,
+        ::AbstractSafeNonlinearTerminationMode)
+    if tc_cache(fx, x, xo)
+        if tc_cache.retcode == NonlinearSafeTerminationReturnCode.Success
+            retcode = ReturnCode.Success
+        elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.PatienceTermination
+            retcode = ReturnCode.ConvergenceFailure
+        elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.ProtectiveTermination
+            retcode = ReturnCode.Unstable
+        else
+            error("Unknown termination code: $(tc_cache.retcode)")
+        end
+        return build_solution(prob, alg, x, fx; retcode)
+    end
+    return nothing
+end
+function check_termination(tc_cache, fx, x, xo, prob, alg,
+        ::AbstractSafeBestNonlinearTerminationMode)
+    if tc_cache(fx, x, xo)
+        if tc_cache.retcode == NonlinearSafeTerminationReturnCode.Success
+            retcode = ReturnCode.Success
+        elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.PatienceTermination
+            retcode = ReturnCode.ConvergenceFailure
+        elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.ProtectiveTermination
+            retcode = ReturnCode.Unstable
+        else
+            error("Unknown termination code: $(tc_cache.retcode)")
+        end
+        if isinplace(prob)
+            prob.f(fx, x, prob.p)
+        else
+            fx = prob.f(x, prob.p)
+        end
+        return build_solution(prob, alg, tc_cache.u, fx; retcode)
+    end
+    return nothing
+end
+
+# MaybeInplace
+@inline __copyto!!(::Number, x) = x
+@inline __copyto!!(::SArray, x) = x
+@inline __copyto!!(y::Union{MArray, Array}, x) = copyto!(y, x)
+@inline function __copyto!!(y::AbstractArray, x)
+    ArrayInterface.can_setindex(y) && return copyto!(y, x)
+    return x
+end
+
+@inline __sub!!(x::Number, Δx) = x - Δx
+@inline __sub!!(x::SArray, Δx) = x .- Δx
+@inline __sub!!(x::Union{MArray, Array}, Δx) = (x .-= Δx)
+@inline function __sub!!(x::AbstractArray, Δx)
+    ArrayInterface.can_setindex(x) && return (x .-= Δx)
+    return x .- Δx
+end
+
+@inline __sub!!(::Number, x, Δx) = x - Δx
+@inline __sub!!(::SArray, x, Δx) = x .- Δx
+@inline __sub!!(y::Union{MArray, Array}, x, Δx) = (@. y = x - Δx)
+@inline function __sub!!(y::AbstractArray, x, Δx)
+    ArrayInterface.can_setindex(y) && return (@. y = x - Δx)
+    return x .- Δx
+end
+
+@inline __add!!(x::Number, Δx) = x + Δx
+@inline __add!!(x::SArray, Δx) = x .+ Δx
+@inline __add!!(x::Union{MArray, Array}, Δx) = (x .+= Δx)
+@inline function __add!!(x::AbstractArray, Δx)
+    ArrayInterface.can_setindex(x) && return (x .+= Δx)
+    return x .+ Δx
+end
+
+@inline __copy(x::Union{Number, SArray}) = x
+@inline __copy(x::Union{Number, SArray}, _) = x
+@inline __copy(x::Union{MArray, Array}) = copy(x)
+@inline __copy(::Union{MArray, Array}, y) = copy(y)
+@inline function __copy(x::AbstractArray)
+    ArrayInterface.can_setindex(x) && return copy(x)
+    return x
+end
+@inline function __copy(x::AbstractArray, y)
+    ArrayInterface.can_setindex(x) && return copy(y)
+    return x
+end
+
+@inline __mul!!(::Union{Number, SArray}, A, b) = A * b
+@inline __mul!!(y::Union{MArray, Array}, A, b) = (mul!(y, A, b); y)
+@inline function __mul!!(y::AbstractArray, A, b)
+    ArrayInterface.can_setindex(y) && return (mul!(y, A, b); y)
+    return A * b
+end
+
+@inline __eval_f(prob, f, fx, x) = isinplace(prob) ? (f(fx, x); fx) : f(x)

From 0915379559c50a3a695ed45e51f891f959320446 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 21 Nov 2023 22:15:53 -0500
Subject: [PATCH 220/375] Update Klement

---
 lib/SimpleNonlinearSolve/Project.toml         |  10 --
 lib/SimpleNonlinearSolve/README.md            |  16 +--
 .../ext/SimpleNonlinearSolveNNlibExt.jl       |  81 ------------
 .../src/SimpleNonlinearSolve.jl               |  13 +-
 .../src/batched/broyden.jl                    |   6 -
 lib/SimpleNonlinearSolve/src/batched/utils.jl |  79 -----------
 lib/SimpleNonlinearSolve/src/broyden.jl       |   1 +
 lib/SimpleNonlinearSolve/src/klement.jl       | 125 ++++++------------
 lib/SimpleNonlinearSolve/src/utils.jl         | 103 ++++++++++-----
 9 files changed, 126 insertions(+), 308 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
 delete mode 100644 lib/SimpleNonlinearSolve/src/batched/broyden.jl
 delete mode 100644 lib/SimpleNonlinearSolve/src/batched/utils.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index f9242c69f..75af93414 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -16,24 +16,14 @@ Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
-[weakdeps]
-NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
-
-[extensions]
-SimpleNonlinearSolveNNlibExt = "NNlib"
-
 [compat]
 ArrayInterface = "7"
 DiffEqBase = "6.126"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 LinearAlgebra = "1.9"
-NNlib = "0.8, 0.9"
 PrecompileTools = "1"
 Reexport = "1"
 SciMLBase = "2.7"
 StaticArraysCore = "1.4"
 julia = "1.9"
-
-[extras]
-NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index efa1fdd63..0f52b1065 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -42,11 +42,11 @@ For more details on the bracketing methods, refer to the [Tutorials](https://doc
 
 ## Breaking Changes in v2
 
-* Batched solvers have been removed in favor of `BatchedArrays.jl`. Stay tuned for detailed
-  tutorials on how to use `BatchedArrays.jl` with `NonlinearSolve` & `SimpleNonlinearSolve`
-  solvers.
-* The old style of specifying autodiff with `chunksize`, `standardtag`, etc. has been
-  deprecated in favor of directly specifying the autodiff type, like `AutoForwardDiff`.
-* `Broyden` and `Klement` have been renamed to `SimpleBroyden` and `SimpleKlement` to
-  avoid conflicts with `NonlinearSolve.jl`'s `GeneralBroyden` and `GeneralKlement`, which
-  will be renamed to `Broyden` and `Klement` in the future.
+  - Batched solvers have been removed in favor of `BatchedArrays.jl`. Stay tuned for detailed
+    tutorials on how to use `BatchedArrays.jl` with `NonlinearSolve` & `SimpleNonlinearSolve`
+    solvers.
+  - The old style of specifying autodiff with `chunksize`, `standardtag`, etc. has been
+    deprecated in favor of directly specifying the autodiff type, like `AutoForwardDiff`.
+  - `Broyden` and `Klement` have been renamed to `SimpleBroyden` and `SimpleKlement` to
+    avoid conflicts with `NonlinearSolve.jl`'s `GeneralBroyden` and `GeneralKlement`, which
+    will be renamed to `Broyden` and `Klement` in the future.
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
deleted file mode 100644
index 1132b64b7..000000000
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveNNlibExt.jl
+++ /dev/null
@@ -1,81 +0,0 @@
-module SimpleNonlinearSolveNNlibExt
-
-using ArrayInterface, DiffEqBase, LinearAlgebra, NNlib, SimpleNonlinearSolve, SciMLBase
-import SimpleNonlinearSolve: _construct_batched_problem_structure,
-    _get_storage, _init_𝓙, _result_from_storage, _get_tolerance, @maybeinplace
-
-function __init__()
-    SimpleNonlinearSolve.NNlibExtLoaded[] = true
-    return
-end
-
-@views function SciMLBase.__solve(prob::NonlinearProblem,
-        alg::BatchedBroyden;
-        abstol = nothing,
-        reltol = nothing,
-        maxiters = 1000,
-        kwargs...)
-    iip = isinplace(prob)
-
-    u, f, reconstruct = _construct_batched_problem_structure(prob)
-    L, N = size(u)
-
-    tc = alg.termination_condition
-    mode = DiffEqBase.get_termination_mode(tc)
-
-    storage = _get_storage(mode, u)
-
-    xₙ, xₙ₋₁, δx, δf = ntuple(_ -> copy(u), 4)
-    T = eltype(u)
-
-    atol = _get_tolerance(abstol, tc.abstol, T)
-    rtol = _get_tolerance(reltol, tc.reltol, T)
-    termination_condition = tc(storage)
-
-    𝓙⁻¹ = _init_𝓙(xₙ)  # L × L × N
-    𝓙⁻¹f, xᵀ𝓙⁻¹δf, xᵀ𝓙⁻¹ = similar(𝓙⁻¹, L, N), similar(𝓙⁻¹, 1, N), similar(𝓙⁻¹, 1, L, N)
-
-    @maybeinplace iip fₙ₋₁=f(xₙ) u
-    iip && (fₙ = copy(fₙ₋₁))
-    for n in 1:maxiters
-        batched_mul!(reshape(𝓙⁻¹f, L, 1, N), 𝓙⁻¹, reshape(fₙ₋₁, L, 1, N))
-        xₙ .= xₙ₋₁ .- 𝓙⁻¹f
-
-        @maybeinplace iip fₙ=f(xₙ)
-        δx .= xₙ .- xₙ₋₁
-        δf .= fₙ .- fₙ₋₁
-
-        batched_mul!(reshape(𝓙⁻¹f, L, 1, N), 𝓙⁻¹, reshape(δf, L, 1, N))
-        δxᵀ = reshape(δx, 1, L, N)
-
-        batched_mul!(reshape(xᵀ𝓙⁻¹δf, 1, 1, N), δxᵀ, reshape(𝓙⁻¹f, L, 1, N))
-        batched_mul!(xᵀ𝓙⁻¹, δxᵀ, 𝓙⁻¹)
-        δx .= (δx .- 𝓙⁻¹f) ./ (xᵀ𝓙⁻¹δf .+ T(1e-5))
-        batched_mul!(𝓙⁻¹, reshape(δx, L, 1, N), xᵀ𝓙⁻¹, one(T), one(T))
-
-        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
-            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
-            return DiffEqBase.build_solution(prob,
-                alg,
-                reconstruct(xₙ),
-                reconstruct(fₙ);
-                retcode)
-        end
-
-        xₙ₋₁ .= xₙ
-        fₙ₋₁ .= fₙ
-    end
-
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        xₙ = storage.u
-        @maybeinplace iip fₙ=f(xₙ)
-    end
-
-    return DiffEqBase.build_solution(prob,
-        alg,
-        reconstruct(xₙ),
-        reconstruct(fₙ);
-        retcode = ReturnCode.MaxIters)
-end
-
-end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 7d04c1037..5ba71e994 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -13,7 +13,7 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
     using FiniteDiff, ForwardDiff
     import ForwardDiff: Dual
     import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace
-    import StaticArraysCore: StaticArray, SVector, SArray, MArray
+    import StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray
 end
 
 @reexport using ADTypes, SciMLBase
@@ -30,7 +30,7 @@ include("utils.jl")
 include("raphson.jl")
 include("broyden.jl")
 # include("lbroyden.jl")
-# include("klement.jl")
+include("klement.jl")
 # include("trustRegion.jl")
 # include("ridder.jl")
 # include("brent.jl")
@@ -41,10 +41,7 @@ include("broyden.jl")
 # include("itp.jl")
 
 # # Batched Solver Support
-# include("batched/utils.jl")
-# include("batched/raphson.jl")
 # include("batched/dfsane.jl")
-# include("batched/broyden.jl")
 
 # ## Default algorithm
 
@@ -88,9 +85,9 @@ include("broyden.jl")
 #     end
 # end
 
-export SimpleBroyden, SimpleGaussNewton, SimpleNewtonRaphson
-# export Bisection, Brent, LBroyden, SimpleDFSane, Falsi, SimpleHalley, Klement,
+export SimpleBroyden, SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson
+# export Bisection, Brent, LBroyden, SimpleDFSane, Falsi, SimpleHalley,
 #     Ridder, SimpleTrustRegion, Alefeld, ITP
-# export BatchedBroyden, BatchedSimpleDFSane
+# export BatchedSimpleDFSane
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/batched/broyden.jl b/lib/SimpleNonlinearSolve/src/batched/broyden.jl
deleted file mode 100644
index ed3cd5dfc..000000000
--- a/lib/SimpleNonlinearSolve/src/batched/broyden.jl
+++ /dev/null
@@ -1,6 +0,0 @@
-struct BatchedBroyden{TC <: NLSolveTerminationCondition} <:
-       AbstractBatchedNonlinearSolveAlgorithm
-    termination_condition::TC
-end
-
-# Implementation of solve using Package Extensions
diff --git a/lib/SimpleNonlinearSolve/src/batched/utils.jl b/lib/SimpleNonlinearSolve/src/batched/utils.jl
deleted file mode 100644
index b8e66fe80..000000000
--- a/lib/SimpleNonlinearSolve/src/batched/utils.jl
+++ /dev/null
@@ -1,79 +0,0 @@
-macro maybeinplace(iip::Symbol, expr::Expr, u0::Union{Symbol, Nothing} = nothing)
-    @assert expr.head == :(=)
-    x1, x2 = expr.args
-    @assert x2.head == :call
-    f, x... = x2.args
-    define_expr = u0 === nothing ? :() : :($(x1) = similar($(u0)))
-    return quote
-        if $(esc(iip))
-            $(esc(define_expr))
-            $(esc(f))($(esc(x1)), $(esc.(x)...))
-        else
-            $(esc(expr))
-        end
-    end
-end
-
-function _get_tolerance(η, tc_η, ::Type{T}) where {T}
-    fallback_η = real(oneunit(T)) * (eps(real(one(T))))^(4 // 5)
-    return ifelse(η !== nothing, η, ifelse(tc_η !== nothing, tc_η, fallback_η))
-end
-
-function _construct_batched_problem_structure(prob)
-    return _construct_batched_problem_structure(prob.u0,
-        prob.f,
-        prob.p,
-        Val(SciMLBase.isinplace(prob)))
-end
-
-function _construct_batched_problem_structure(u0::AbstractArray{T, N},
-        f,
-        p,
-        ::Val{iip}) where {T, N, iip}
-    # Reconstruct `u`
-    reconstruct = N == 2 ? identity : Base.Fix2(reshape, size(u0))
-    # Standardize `u`
-    standardize = N == 2 ? identity :
-                  (N == 1 ? Base.Fix2(reshape, (:, 1)) :
-                   Base.Fix2(reshape, (:, size(u0, ndims(u0)))))
-    # Updated Function
-    f_modified = if iip
-        function f_modified_iip(du, u)
-            f(reconstruct(du), reconstruct(u), p)
-            return standardize(du)
-        end
-    else
-        f_modified_oop(u) = standardize(f(reconstruct(u), p))
-    end
-    return standardize(u0), f_modified, reconstruct
-end
-
-@views function _init_𝓙(x::AbstractMatrix)
-    𝓙 = ArrayInterface.zeromatrix(x[:, 1])
-    if ismutable(x)
-        𝓙[diagind(𝓙)] .= one(eltype(x))
-    else
-        𝓙 .+= I
-    end
-    return repeat(𝓙, 1, 1, size(x, 2))
-end
-
-_result_from_storage(::Nothing, xₙ, fₙ, args...) = ReturnCode.Success, xₙ, fₙ
-function _result_from_storage(storage::NLSolveSafeTerminationResult, xₙ, fₙ, f, mode, iip)
-    if storage.return_code == DiffEqBase.NLSolveSafeTerminationReturnCode.Success
-        return ReturnCode.Success, xₙ, fₙ
-    else
-        if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-            @maybeinplace iip fₙ=f(xₙ)
-            return ReturnCode.Terminated, storage.u, fₙ
-        else
-            return ReturnCode.Terminated, xₙ, fₙ
-        end
-    end
-end
-
-function _get_storage(mode, u)
-    return mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ?
-           NLSolveSafeTerminationResult(mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES ? u :
-                                        nothing) : nothing
-end
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 4b7d5d902..7587168a4 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -32,6 +32,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
         tc_sol !== nothing && return tc_sol
 
         J⁻¹δf = _restructure(J⁻¹δf, __mul!!(_vec(J⁻¹δf), J⁻¹, _vec(δf)))
+        δx = __neg!!(δx)
         d = dot(δx, J⁻¹δf)
         xᵀJ⁻¹ = _restructure(xᵀJ⁻¹, __mul!!(_vec(xᵀJ⁻¹), _vec(δx)', J⁻¹))
 
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index e6a38ecc2..3d22d1c07 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -1,106 +1,69 @@
 """
-```julia
-Klement()
-```
+    SimpleKlement()
 
 A low-overhead implementation of [Klement](https://jatm.com.br/jatm/article/view/373).
-This method is non-allocating on scalar problems.
 """
-struct Klement <: AbstractSimpleNonlinearSolveAlgorithm end
+struct SimpleKlement <: AbstractSimpleNonlinearSolveAlgorithm end
 
-function SciMLBase.__solve(prob::NonlinearProblem,
-        alg::Klement, args...; abstol = nothing,
-        reltol = nothing,
-        maxiters = 1000, kwargs...)
-    f = Base.Fix2(prob.f, prob.p)
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        termination_condition = nothing, kwargs...)
+    f = isinplace(prob) ? (du, u) -> prob.f(du, u, prob.p) : u -> prob.f(u, prob.p)
     x = float(prob.u0)
-    fₙ = f(x)
     T = eltype(x)
-    singular_tol = 1e-9
+    fx = _get_fx(prob, x)
 
-    if SciMLBase.isinplace(prob)
-        error("Klement currently only supports out-of-place nonlinear problems")
-    end
-
-    atol = abstol !== nothing ? abstol :
-           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
-    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
-
-    xₙ = x
-    xₙ₋₁ = x
-    fₙ₋₁ = fₙ
-
-    # x is scalar
-    if x isa Number
-        J = 1.0
-        for _ in 1:maxiters
-            xₙ = xₙ₋₁ - fₙ₋₁ / J
-            fₙ = f(xₙ)
-
-            iszero(fₙ) &&
-                return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                    retcode = ReturnCode.Success)
-
-            if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
-                return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                    retcode = ReturnCode.Success)
-            end
+    singular_tol = eps(T)^(2 // 3)
 
-            Δxₙ = xₙ - xₙ₋₁
-            Δfₙ = fₙ - fₙ₋₁
+    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+        termination_condition)
 
-            # Prevent division by 0
-            denominator = max(J^2 * Δxₙ^2, 1e-9)
+    δx, fprev, xo, δf, d = __copy(fx), __copy(fx), __copy(x), __copy(fx), __copy(x)
+    J = __init_identity_jacobian(fx, x)
+    J_cache, δx² = __copy(J), __copy(x)
 
-            k = (Δfₙ - J * Δxₙ) / denominator
-            J += (k * Δxₙ * J) * J
+    for _ in 1:maxiters
+        if x isa Number
+            J < singular_tol && (J = __init_identity_jacobian!!(J))
+            F = J
+        else
+            F = lu(J; check = false)
 
             # Singularity test
-            if J < singular_tol
-                J = 1.0
+            if any(x -> abs(x) < singular_tol, @view(F.U[diagind(F.U)]))
+                J = __init_identity_jacobian!!(J)
+                F = lu(J; check = false)
             end
-
-            xₙ₋₁ = xₙ
-            fₙ₋₁ = fₙ
         end
-        # x is a vector
-    else
-        J = init_J(x)
-        for _ in 1:maxiters
-            F = lu(J, check = false)
-
-            # Singularity test
-            if any(abs.(F.U[diagind(F.U)]) .< singular_tol)
-                J = init_J(xₙ)
-                F = lu(J, check = false)
-            end
 
-            tmp = _restructure(fₙ₋₁, F \ _vec(fₙ₋₁))
-            xₙ = xₙ₋₁ - tmp
-            fₙ = f(xₙ)
+        δx = __copyto!!(δx, fprev)
+        δx = __ldiv!!(F, δx)
+        x = __sub!!(x, xo, δx)
+        fx = __eval_f(prob, f, fx, x)
 
-            iszero(fₙ) &&
-                return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                    retcode = ReturnCode.Success)
+        # Termination Checks
+        tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+        tc_sol !== nothing && return tc_sol
 
-            if isapprox(xₙ, xₙ₋₁, atol = atol, rtol = rtol)
-                return SciMLBase.build_solution(prob, alg, xₙ, fₙ;
-                    retcode = ReturnCode.Success)
-            end
+        δx = __neg!!(δx)
+        δf = __sub!!(δf, fx, fprev)
 
-            Δxₙ = xₙ - xₙ₋₁
-            Δfₙ = fₙ - fₙ₋₁
+        # Prevent division by 0
+        δx² = __broadcast!!(δx², abs2, δx)
+        J_cache = __broadcast!!(J_cache, abs2, J)
+        d = _restructure(d, __mul!!(_vec(d), J_cache', _vec(δx²)))
+        d = __broadcast!!(d, Base.Fix2(max, singular_tol), d)
 
-            # Prevent division by 0
-            denominator = _restructure(Δxₙ, max.(J' .^ 2 * _vec(Δxₙ) .^ 2, 1e-9))
+        δx² = _restructure(δx², __mul!!(_vec(δx²), J, _vec(δx)))
+        δf = __sub!!(δf, δx²)
+        δf = __broadcast!!(δf, /, δf, d)
 
-            k = (Δfₙ - _restructure(Δxₙ, J * _vec(Δxₙ))) ./ denominator
-            J += (_vec(k) * _vec(Δxₙ)' .* J) * J
+        J_cache = __mul!!(J_cache, _vec(δf), _vec(δx)')
+        J_cache = __broadcast!!(J_cache, *, J_cache, J)
+        J_cache = __mul!!(J_cache, J_cache, J)
 
-            xₙ₋₁ = xₙ
-            fₙ₋₁ = fₙ
-        end
+        J = __add!!(J, J_cache)
     end
 
-    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+    return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 6a35aae25..228006409 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -22,9 +22,6 @@ end
 #     x1 > x0 ? max(a, b) : min(a, b)
 # end
 
-# alg_autodiff(alg::AbstractNewtonAlgorithm{CS, AD, FDT}) where {CS, AD, FDT} = AD
-# diff_type(alg::AbstractNewtonAlgorithm{CS, AD, FDT}) where {CS, AD, FDT} = FDT
-
 __standard_tag(::Nothing, x) = ForwardDiff.Tag(SimpleNonlinearSolveTag(), eltype(x))
 __standard_tag(tag::ForwardDiff.Tag, _) = tag
 __standard_tag(tag, x) = ForwardDiff.Tag(tag, eltype(x))
@@ -86,6 +83,26 @@ function value_and_jacobian(ad, f::F, y, x::X, p, cache; J = nothing) where {F,
     end
 end
 
+function value_and_jacobian(ad, f::F, y, x::Number, p, cache; J = nothing) where {F}
+    if DiffEqBase.has_jac(f)
+        return f(x, p), f.jac(x, p)
+    elseif ad isa AutoForwardDiff
+        T = typeof(__standard_tag(ad.tag, x))
+        out = f(ForwardDiff.Dual{T}(x, one(x)), p)
+        return ForwardDiff.value(out), ForwardDiff.extract_derivative(T, out)
+    elseif ad isa AutoFiniteDiff
+        _f = Base.Fix2(f, p)
+        return _f(x), FiniteDiff.finite_difference_derivative(_f, x, ad.fdtype)
+    else
+        throw(ArgumentError("Unsupported AD method: $(ad)"))
+    end
+end
+
+"""
+    jacobian_cache(ad, f, y, x, p) --> J, cache
+
+Returns a Jacobian Matrix and a cache for the Jacobian computation.
+"""
 function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
     if isinplace(f)
         _f = (du, u) -> f(du, u, p)
@@ -114,45 +131,29 @@ function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
     end
 end
 
-# """
-#   value_derivative(f, x)
-
-# Compute `f(x), d/dx f(x)` in the most efficient way.
-# """
-# function value_derivative(f::F, x::R) where {F, R}
-#     T = typeof(ForwardDiff.Tag(f, R))
-#     out = f(ForwardDiff.Dual{T}(x, one(x)))
-#     ForwardDiff.value(out), ForwardDiff.extract_derivative(T, out)
-# end
-# value_derivative(f::F, x::AbstractArray) where {F} = f(x), ForwardDiff.jacobian(f, x)
-
-# """
-#     value_derivative!(J, y, f!, x, cfg = JacobianConfig(f!, y, x))
-
-# Inplace version of [`SimpleNonlinearSolve.value_derivative`](@ref).
-# """
-# function value_derivative!(J::AbstractMatrix,
-#     y::AbstractArray,
-#     f!::F,
-#     x::AbstractArray,
-#     cfg::ForwardDiff.JacobianConfig = ForwardDiff.JacobianConfig(f!, y, x)) where {F}
-#     ForwardDiff.jacobian!(J, f!, y, x, cfg)
-#     return y, J
-# end
-
-# value(x) = x
-# value(x::Dual) = ForwardDiff.value(x)
-# value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
+jacobian_cache(ad, f::F, y, x::Number, p) where {F} = nothing, nothing
 
-__init_identity_jacobian(u::Number, _) = u
+__init_identity_jacobian(u::Number, _) = one(u)
+__init_identity_jacobian!!(J::Number) = one(J)
 function __init_identity_jacobian(u, fu)
     J = similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u))
     J[diagind(J)] .= one(eltype(J))
     return J
 end
+function __init_identity_jacobian!!(J)
+    fill!(J, zero(eltype(J)))
+    J[diagind(J)] .= one(eltype(J))
+    return J
+end
 function __init_identity_jacobian(u::StaticArray, fu)
-    return convert(MArray{Tuple{length(fu), length(u)}},
-        Matrix{eltype(u)}(I, length(fu), length(u)))
+    S1, S2 = length(fu), length(u)
+    J = SMatrix{S1, S2, eltype(u)}(ntuple(i -> ifelse(i ∈ 1:(S1 + 1):(S1 * S2), 1, 0),
+        S1 * S2))
+    return J
+end
+function __init_identity_jacobian!!(J::StaticArray{S1, S2}) where {S1, S2}
+    return SMMatrix{S1, S2, eltype(J)}(ntuple(i -> ifelse(i ∈ 1:(S1 + 1):(S1 * S2), 1, 0),
+        S1 * S2))
 end
 
 # function dogleg_method(J, f, g, Δ)
@@ -300,6 +301,14 @@ end
     return x .+ Δx
 end
 
+@inline __add!!(::Number, x, Δx) = x + Δx
+@inline __add!!(::SArray, x, Δx) = x .+ Δx
+@inline __add!!(y::Union{MArray, Array}, x, Δx) = (@. y = x + Δx)
+@inline function __add!!(y::AbstractArray, x, Δx)
+    ArrayInterface.can_setindex(y) && return (@. y = x + Δx)
+    return x .+ Δx
+end
+
 @inline __copy(x::Union{Number, SArray}) = x
 @inline __copy(x::Union{Number, SArray}, _) = x
 @inline __copy(x::Union{MArray, Array}) = copy(x)
@@ -320,4 +329,28 @@ end
     return A * b
 end
 
+@inline __neg!!(x::Union{Number, SArray}) = -x
+@inline __neg!!(x::Union{MArray, Array}) = (@. x .*= -one(eltype(x)))
+@inline function __neg!!(x::AbstractArray)
+    ArrayInterface.can_setindex(x) && return (@. x .*= -one(eltype(x)))
+    return -x
+end
+
+@inline __ldiv!!(A, b::Union{Number, SArray}) = A \ b
+@inline __ldiv!!(A, b::Union{MArray, Array}) = (ldiv!(A, b); b)
+@inline function __ldiv!!(A, b::AbstractArray)
+    ArrayInterface.can_setindex(b) && return (ldiv!(A, b); b)
+    return A \ b
+end
+
+@inline __broadcast!!(y::Union{Number, SArray}, f::F, x, args...) where {F} = f.(x, args...)
+@inline function __broadcast!!(y::Union{MArray, Array}, f::F, x, args...) where {F}
+    @. y = f(x, args...)
+    return y
+end
+@inline function __broadcast!!(y::AbstractArray, f::F, x, args...) where {F}
+    ArrayInterface.can_setindex(y) && return (@. y = f(x, args...))
+    return f.(x, args...)
+end
+
 @inline __eval_f(prob, f, fx, x) = isinplace(prob) ? (f(fx, x); fx) : f(x)

From d3af9e1eb533c0b237883e284129a639238fb558 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 21 Nov 2023 22:41:08 -0500
Subject: [PATCH 221/375] CLeanup bisection

---
 .../src/SimpleNonlinearSolve.jl               |  19 ++-
 .../src/batched/dfsane.jl                     | 141 ------------------
 lib/SimpleNonlinearSolve/src/bisection.jl     |  83 +++--------
 3 files changed, 35 insertions(+), 208 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/src/batched/dfsane.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 5ba71e994..8564a2805 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -25,23 +25,26 @@ abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorit
 abstract type AbstractNewtonAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 
 include("utils.jl")
-# include("bisection.jl")
-# include("falsi.jl")
+
+# Nonlinear Solvera
 include("raphson.jl")
 include("broyden.jl")
 # include("lbroyden.jl")
 include("klement.jl")
 # include("trustRegion.jl")
+# include("halley.jl")
+# include("dfsane.jl")
+
+# Interval Nonlinear Solvers
+include("bisection.jl")
+# include("falsi.jl")
 # include("ridder.jl")
 # include("brent.jl")
-# include("dfsane.jl")
-# include("ad.jl")
-# include("halley.jl")
 # include("alefeld.jl")
 # include("itp.jl")
 
-# # Batched Solver Support
-# include("batched/dfsane.jl")
+# AD
+# include("ad.jl")
 
 # ## Default algorithm
 
@@ -86,8 +89,8 @@ include("klement.jl")
 # end
 
 export SimpleBroyden, SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson
+export Bisection
 # export Bisection, Brent, LBroyden, SimpleDFSane, Falsi, SimpleHalley,
 #     Ridder, SimpleTrustRegion, Alefeld, ITP
-# export BatchedSimpleDFSane
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl b/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
deleted file mode 100644
index 01b3b1996..000000000
--- a/lib/SimpleNonlinearSolve/src/batched/dfsane.jl
+++ /dev/null
@@ -1,141 +0,0 @@
-Base.@kwdef struct BatchedSimpleDFSane{T, F, TC <: NLSolveTerminationCondition} <:
-                   AbstractBatchedNonlinearSolveAlgorithm
-    σₘᵢₙ::T = 1.0f-10
-    σₘₐₓ::T = 1.0f+10
-    σ₁::T = 1.0f0
-    M::Int = 10
-    γ::T = 1.0f-4
-    τₘᵢₙ::T = 0.1f0
-    τₘₐₓ::T = 0.5f0
-    nₑₓₚ::Int = 2
-    ηₛ::F = (f₍ₙₒᵣₘ₎₁, n, xₙ, fₙ) -> f₍ₙₒᵣₘ₎₁ ./ n .^ 2
-    termination_condition::TC = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-        abstol = nothing,
-        reltol = nothing)
-    max_inner_iterations::Int = 1000
-end
-
-function SciMLBase.__solve(prob::NonlinearProblem,
-        alg::BatchedSimpleDFSane,
-        args...;
-        abstol = nothing,
-        reltol = nothing,
-        maxiters = 100,
-        kwargs...)
-    iip = isinplace(prob)
-
-    u, f, reconstruct = _construct_batched_problem_structure(prob)
-    L, N = size(u)
-    T = eltype(u)
-
-    tc = alg.termination_condition
-    mode = DiffEqBase.get_termination_mode(tc)
-
-    storage = _get_storage(mode, u)
-
-    atol = _get_tolerance(abstol, tc.abstol, T)
-    rtol = _get_tolerance(reltol, tc.reltol, T)
-    termination_condition = tc(storage)
-
-    σₘᵢₙ, σₘₐₓ, γ, τₘᵢₙ, τₘₐₓ = T(alg.σₘᵢₙ), T(alg.σₘₐₓ), T(alg.γ), T(alg.τₘᵢₙ), T(alg.τₘₐₓ)
-    α₁ = one(T)
-    α₊, α₋ = similar(u, 1, N), similar(u, 1, N)
-    σₙ = fill(T(alg.σ₁), 1, N)
-    𝒹 = similar(σₙ, L, N)
-    M = alg.M
-    nₑₓₚ = alg.nₑₓₚ
-
-    xₙ, xₙ₋₁, f₍ₙₒᵣₘ₎ₙ₋₁, f₍ₙₒᵣₘ₎ₙ = copy(u), copy(u), similar(u, 1, N), similar(u, 1, N)
-
-    function ff!(fₓ, fₙₒᵣₘ, x)
-        f(fₓ, x)
-        sum!(abs2, fₙₒᵣₘ, fₓ)
-        fₙₒᵣₘ .^= (nₑₓₚ / 2)
-        return fₓ
-    end
-
-    function ff!(fₙₒᵣₘ, x)
-        fₓ = f(x)
-        sum!(abs2, fₙₒᵣₘ, fₓ)
-        fₙₒᵣₘ .^= (nₑₓₚ / 2)
-        return fₓ
-    end
-
-    @maybeinplace iip fₙ₋₁=ff!(f₍ₙₒᵣₘ₎ₙ₋₁, xₙ) xₙ
-    iip && (fₙ = similar(fₙ₋₁))
-    ℋ = repeat(f₍ₙₒᵣₘ₎ₙ₋₁, M, 1)
-    f̄ = similar(ℋ, 1, N)
-    ηₛ = (n, xₙ, fₙ) -> alg.ηₛ(f₍ₙₒᵣₘ₎ₙ₋₁, n, xₙ, fₙ)
-
-    for n in 1:maxiters
-        # Spectral parameter range check
-        @. σₙ = sign(σₙ) * clamp(abs(σₙ), σₘᵢₙ, σₘₐₓ)
-
-        # Line search direction
-        @. 𝒹 = -σₙ * fₙ₋₁
-
-        η = ηₛ(n, xₙ₋₁, fₙ₋₁)
-        maximum!(f̄, ℋ)
-        fill!(α₊, α₁)
-        fill!(α₋, α₁)
-        @. xₙ = xₙ₋₁ + α₊ * 𝒹
-
-        @maybeinplace iip fₙ=ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
-
-        for _ in 1:(alg.max_inner_iterations)
-            𝒸 = @. f̄ + η - γ * α₊^2 * f₍ₙₒᵣₘ₎ₙ₋₁
-
-            (sum(f₍ₙₒᵣₘ₎ₙ .≤ 𝒸) ≥ N ÷ 2) && break
-
-            @. α₊ = clamp(α₊^2 * f₍ₙₒᵣₘ₎ₙ₋₁ / (f₍ₙₒᵣₘ₎ₙ + (T(2) * α₊ - T(1)) * f₍ₙₒᵣₘ₎ₙ₋₁),
-                τₘᵢₙ * α₊,
-                τₘₐₓ * α₊)
-            @. xₙ = xₙ₋₁ - α₋ * 𝒹
-            @maybeinplace iip fₙ=ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
-
-            (sum(f₍ₙₒᵣₘ₎ₙ .≤ 𝒸) ≥ N ÷ 2) && break
-
-            @. α₋ = clamp(α₋^2 * f₍ₙₒᵣₘ₎ₙ₋₁ / (f₍ₙₒᵣₘ₎ₙ + (T(2) * α₋ - T(1)) * f₍ₙₒᵣₘ₎ₙ₋₁),
-                τₘᵢₙ * α₋,
-                τₘₐₓ * α₋)
-            @. xₙ = xₙ₋₁ + α₊ * 𝒹
-            @maybeinplace iip fₙ=ff!(f₍ₙₒᵣₘ₎ₙ, xₙ)
-        end
-
-        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
-            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
-            return DiffEqBase.build_solution(prob,
-                alg,
-                reconstruct(xₙ),
-                reconstruct(fₙ);
-                retcode)
-        end
-
-        # Update spectral parameter
-        @. xₙ₋₁ = xₙ - xₙ₋₁
-        @. fₙ₋₁ = fₙ - fₙ₋₁
-
-        sum!(abs2, α₊, xₙ₋₁)
-        sum!(α₋, xₙ₋₁ .* fₙ₋₁)
-        σₙ .= α₊ ./ (α₋ .+ T(1e-5))
-
-        # Take step
-        @. xₙ₋₁ = xₙ
-        @. fₙ₋₁ = fₙ
-        @. f₍ₙₒᵣₘ₎ₙ₋₁ = f₍ₙₒᵣₘ₎ₙ
-
-        # Update history
-        ℋ[n % M + 1, :] .= view(f₍ₙₒᵣₘ₎ₙ, 1, :)
-    end
-
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        xₙ = storage.u
-        @maybeinplace iip fₙ=f(xₙ)
-    end
-
-    return DiffEqBase.build_solution(prob,
-        alg,
-        reconstruct(xₙ),
-        reconstruct(fₙ);
-        retcode = ReturnCode.MaxIters)
-end
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 93b1cbeb0..7e8404451 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -1,5 +1,5 @@
 """
-`Bisection(; exact_left = false, exact_right = false)`
+    Bisection(; exact_left = false, exact_right = false)
 
 A common bisection method.
 
@@ -10,83 +10,48 @@ A common bisection method.
   - `exact_right`: whether to enforce whether the right side of the interval must be exactly
     zero for the returned result. Defaults to false.
 """
-struct Bisection <: AbstractBracketingAlgorithm
-    exact_left::Bool
-    exact_right::Bool
-end
-
-function Bisection(; exact_left = false, exact_right = false)
-    Bisection(exact_left, exact_right)
+@kwdef struct Bisection <: AbstractBracketingAlgorithm
+    exact_left::Bool = false
+    exact_right::Bool = false
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...;
         maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
         kwargs...)
+    @assert !isinplace(prob) "Bisection only supports OOP problems."
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
+
     if iszero(fl)
-        return SciMLBase.build_solution(prob, alg, left, fl;
-            retcode = ReturnCode.ExactSolutionLeft, left = left,
-            right = right)
+        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
+            left, right)
     end
+
     if iszero(fr)
-        return SciMLBase.build_solution(prob, alg, right, fr;
-            retcode = ReturnCode.ExactSolutionRight, left = left,
-            right = right)
+        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+            left, right)
     end
 
-    i = 1
-    if !iszero(fr)
-        while i < maxiters
-            mid = (left + right) / 2
-            (mid == left || mid == right) &&
-                return SciMLBase.build_solution(prob, alg, left, fl;
-                    retcode = ReturnCode.FloatingPointLimit,
-                    left = left, right = right)
-            fm = f(mid)
-            if abs((right - left) / 2) < abstol
-                return SciMLBase.build_solution(prob, alg, mid, fm;
-                    retcode = ReturnCode.Success,
-                    left = left, right = right)
-            end
-            if iszero(fm)
-                right = mid
-                break
-            end
-            if sign(fl) == sign(fm)
-                fl = fm
-                left = mid
-            else
-                fr = fm
-                right = mid
-            end
-            i += 1
+    for _ in 1:maxiters
+        mid = (left + right) / 2
+        if (mid == left || mid == right)
+            return build_solution(prob, alg, left, fl; left, right,
+                retcode = ReturnCode.FloatingPointLimit)
         end
-    end
 
-    while i < maxiters
-        mid = (left + right) / 2
-        (mid == left || mid == right) &&
-            return SciMLBase.build_solution(prob, alg, left, fl;
-                retcode = ReturnCode.FloatingPointLimit,
-                left = left, right = right)
         fm = f(mid)
-        if abs((right - left) / 2) < abstol
-            return SciMLBase.build_solution(prob, alg, mid, fm;
-                retcode = ReturnCode.Success,
-                left = left, right = right)
+        if abs((right - left) / 2) < abstol || iszero(fm)
+            return build_solution(prob, alg, mid, fm; left, right,
+                retcode = ReturnCode.Success)
         end
-        if iszero(fm)
-            right = mid
-            fr = fm
+
+        if sign(fl * fm) < 0
+            right, fr = mid, fm
         else
-            left = mid
-            fl = fm
+            left, fl = mid, fm
         end
-        i += 1
     end
 
-    return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
-        left = left, right = right)
+    return build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
 end

From 71ed2fe54951aaf7fc008292150d08543136a7be Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 22 Nov 2023 02:44:43 -0500
Subject: [PATCH 222/375] Make some progress on Falsi and SimpleDFSane

---
 .../src/SimpleNonlinearSolve.jl               |  10 +-
 lib/SimpleNonlinearSolve/src/bisection.jl     |  14 +-
 lib/SimpleNonlinearSolve/src/dfsane.jl        | 199 +++++++-----------
 lib/SimpleNonlinearSolve/src/falsi.jl         | 117 +++++-----
 lib/SimpleNonlinearSolve/src/trustRegion.jl   |  91 ++++----
 lib/SimpleNonlinearSolve/src/utils.jl         |  39 ++--
 6 files changed, 221 insertions(+), 249 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 8564a2805..e0293ee6c 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -33,11 +33,11 @@ include("broyden.jl")
 include("klement.jl")
 # include("trustRegion.jl")
 # include("halley.jl")
-# include("dfsane.jl")
+include("dfsane.jl")
 
 # Interval Nonlinear Solvers
 include("bisection.jl")
-# include("falsi.jl")
+include("falsi.jl")
 # include("ridder.jl")
 # include("brent.jl")
 # include("alefeld.jl")
@@ -88,9 +88,9 @@ include("bisection.jl")
 #     end
 # end
 
-export SimpleBroyden, SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson
-export Bisection
-# export Bisection, Brent, LBroyden, SimpleDFSane, Falsi, SimpleHalley,
+export SimpleBroyden, SimpleDFSane, SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson
+export Bisection, Falsi
+# export Bisection, Brent, LBroyden, SimpleHalley,
 #     Ridder, SimpleTrustRegion, Alefeld, ITP
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 7e8404451..9b1394bc6 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -9,6 +9,10 @@ A common bisection method.
     zero for the returned result. Defaults to false.
   - `exact_right`: whether to enforce whether the right side of the interval must be exactly
     zero for the returned result. Defaults to false.
+
+!!! warning
+
+    Currently, the keyword arguments are not implemented.
 """
 @kwdef struct Bisection <: AbstractBracketingAlgorithm
     exact_left::Bool = false
@@ -16,13 +20,15 @@ A common bisection method.
 end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...;
-        maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-        kwargs...)
-    @assert !isinplace(prob) "Bisection only supports OOP problems."
+        maxiters = 1000, abstol = nothing, kwargs...)
+    @assert !isinplace(prob) "`Bisection` only supports OOP problems."
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
+    abstol = _get_tolerance(abstol,
+        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+
     if iszero(fl)
         return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
             left, right)
@@ -41,7 +47,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
         end
 
         fm = f(mid)
-        if abs((right - left) / 2) < abstol || iszero(fm)
+        if abs((right - left) / 2) < abstol || abs(fm) < abstol
             return build_solution(prob, alg, mid, fm; left, right,
                 retcode = ReturnCode.Success)
         end
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index e7fda8629..d646171d5 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -1,12 +1,7 @@
 """
     SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
         M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
-        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2,
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing),
-        batched::Bool = false,
-        max_inner_iterations::Int = 1000)
+        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2)
 
 A low-overhead implementation of the df-sane method for solving large-scale nonlinear
 systems of equations. For in depth information about all the parameters and the algorithm,
@@ -42,167 +37,133 @@ Computation, 75, 1429-1448.](https://www.researchgate.net/publication/220576479_
     ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and
     `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to
     ``||F||^2 / k^2``.
-  - `termination_condition`: a `NLSolveTerminationCondition` that determines when the solver
-    should terminate. Defaults to `NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault; abstol = nothing, reltol = nothing)`.
-  - `batched`: if `true`, the algorithm will use a batched version of the algorithm that treats each
-    column of `x` as a separate problem. This can be useful nonlinear problems involing neural
-    networks. Defaults to `false`.
-  - `max_inner_iterations`: the maximum number of iterations allowed for the inner loop of the
-    algorithm. Used exclusively in `batched` mode. Defaults to `1000`.
 """
-struct SimpleDFSane{T, TC} <: AbstractSimpleNonlinearSolveAlgorithm
-    σ_min::T
-    σ_max::T
-    σ_1::T
-    M::Int
-    γ::T
-    τ_min::T
-    τ_max::T
-    nexp::Int
-    η_strategy::Function
-    termination_condition::TC
+@kwdef @concrete struct SimpleDFSane <: AbstractSimpleNonlinearSolveAlgorithm
+    σ_min = 1e-10
+    σ_max = 1e10
+    σ_1 = 1.0
+    M::Int = 10
+    γ = 1e-4
+    τ_min = 0.1
+    τ_max = 0.5
+    nexp::Int = 2
+    η_strategy = (f_1, k, x, F) -> f_1 ./ k^2
 end
 
-function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
-        M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
-        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2,
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing),
-        batched::Bool = false,
-        max_inner_iterations = 1000)
-    if batched
-        return BatchedSimpleDFSane(; σₘᵢₙ = σ_min,
-            σₘₐₓ = σ_max,
-            σ₁ = σ_1,
-            M,
-            γ,
-            τₘᵢₙ = τ_min,
-            τₘₐₓ = τ_max,
-            nₑₓₚ = nexp,
-            ηₛ = η_strategy,
-            termination_condition,
-            max_inner_iterations)
-    end
-    return SimpleDFSane{typeof(σ_min), typeof(termination_condition)}(σ_min,
-        σ_max,
-        σ_1,
-        M,
-        γ,
-        τ_min,
-        τ_max,
-        nexp,
-        η_strategy,
-        termination_condition)
-end
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        termination_condition = nothing, kwargs...)
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane,
-        args...; abstol = nothing, reltol = nothing, maxiters = 1000,
-        kwargs...)
-    tc = alg.termination_condition
-    mode = DiffEqBase.get_termination_mode(tc)
+    f = isinplace(prob) ? (du, u) -> prob.f(du, u, prob.p) : u -> prob.f(u, prob.p)
 
-    f = Base.Fix2(prob.f, prob.p)
     x = float(prob.u0)
-
+    fx = _get_fx(prob, x)
     T = eltype(x)
-    σ_min = float(alg.σ_min)
-    σ_max = float(alg.σ_max)
-    σ_k = float(alg.σ_1)
+
+    σ_min = T(alg.σ_min)
+    σ_max = T(alg.σ_max)
+    σ_k = T(alg.σ_1)
 
     M = alg.M
-    γ = float(alg.γ)
-    τ_min = float(alg.τ_min)
-    τ_max = float(alg.τ_max)
+    γ = T(alg.γ)
+    τ_min = T(alg.τ_min)
+    τ_max = T(alg.τ_max)
     nexp = alg.nexp
     η_strategy = alg.η_strategy
 
-    if SciMLBase.isinplace(prob)
-        error("SimpleDFSane currently only supports out-of-place nonlinear problems")
-    end
-
-    atol = _get_tolerance(abstol, tc.abstol, T)
-    rtol = _get_tolerance(reltol, tc.reltol, T)
-
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        error("SimpleDFSane currently doesn't support SAFE_BEST termination modes")
-    end
-
-    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
-              nothing
-    termination_condition = tc(storage)
+    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+        termination_condition)
 
-    function ff(x)
-        F = f(x)
-        f_k = norm(F)^nexp
-        return f_k, F
+    ff = if isinplace(prob)
+        function (_fx, x)
+            f(_fx, x)
+            f_k = norm(_fx)^nexp
+            return f_k, _fx
+        end
+    else
+        function (x)
+            _fx = f(x)
+            f_k = norm(_fx)^nexp
+            return f_k, _fx
+        end
     end
 
-    function generate_history(f_k, M)
-        return fill(f_k, M)
-    end
+    generate_history(f_k, M) = fill(f_k, M)
 
-    f_k, F_k = ff(x)
-    α_1 = convert(T, 1.0)
+    f_k, F_k = isinplace(prob) ? ff(fx, x) : ff(x)
+    F_k = __copy(F_k)
+    α_1 = one(T)
     f_1 = f_k
     history_f_k = generate_history(f_k, M)
 
+    # Generate the cache
+    d, xo, x_cache, δx, δf = __copy(x), __copy(x), __copy(x), __copy(x), __copy(x)
+    α_tp, α_tm = __copy(x), __copy(x)
+
     for k in 1:maxiters
         # Spectral parameter range check
         σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
 
         # Line search direction
-        d = -σ_k .* F_k
+        d = __broadcast!!(d, *, -σ_k, F_k)
 
         η = η_strategy(f_1, k, x, F_k)
         f̄ = maximum(history_f_k)
         α_p = α_1
         α_m = α_1
-        x_new = @. x + α_p * d
 
-        f_new, F_new = ff(x_new)
+        x_cache = __broadcast!!(x_cache, *, α_p, d)
+        x = __broadcast!!(x, +, x_cache)
 
-        inner_iterations = 0
-        while true
-            inner_iterations += 1
+        f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
 
+        # FIXME: This part is not correctly implemented
+        while true
             criteria = f̄ + η - γ * α_p^2 * f_k
             f_new ≤ criteria && break
 
-            α_tp = @. α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
-            x_new = @. x - α_m * d
-            f_new, F_new = ff(x_new)
+            if ArrayInterface.can_setindex(α_tp) && !(x isa Number)
+                @. α_tp = α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
+            else
+                α_tp = @. α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
+            end
+            x_cache = __broadcast!!(x_cache, *, α_m, d)
+            x = __broadcast!!(x, -, x_cache)
+            f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
 
             f_new ≤ criteria && break
 
-            α_tm = @. α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
-            α_p = @. clamp(α_tp, τ_min * α_p, τ_max * α_p)
-            α_m = @. clamp(α_tm, τ_min * α_m, τ_max * α_m)
-            x_new = @. x + α_p * d
-            f_new, F_new = ff(x_new)
+            if ArrayInterface.can_setindex(α_tm) && !(x isa Number)
+                @. α_tm = α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
+                @. α_p = clamp(α_tp, τ_min * α_p, τ_max * α_p)
+                @. α_m = clamp(α_tm, τ_min * α_m, τ_max * α_m)
+            else
+                α_tm = @. α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
+                α_p = @. clamp(α_tp, τ_min * α_p, τ_max * α_p)
+                α_m = @. clamp(α_tm, τ_min * α_m, τ_max * α_m)
+            end
+            x_cache = __broadcast!!(x_cache, *, α_p, d)
+            x = __broadcast!!(x, +, x_cache)
+            f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
         end
 
-        if termination_condition(F_new, x_new, x, atol, rtol)
-            return SciMLBase.build_solution(prob,
-                alg,
-                x_new,
-                F_new;
-                retcode = ReturnCode.Success)
-        end
+        tc_sol = check_termination(tc_cache, f_new, x, xo, prob, alg)
+        tc_sol !== nothing && return tc_sol
 
         # Update spectral parameter
-        s_k = @. x_new - x
-        y_k = @. F_new - F_k
+        δx = __broadcast!!(δx, -, x, xo)
+        δf = __broadcast!!(δf, -, F_new, F_k)
 
-        σ_k = (s_k' * s_k) / (s_k' * y_k)
+        σ_k = dot(δx, δx) / dot(δx, δf)
 
         # Take step
-        x = x_new
-        F_k = F_new
+        xo = __copyto!!(xo, x)
+        F_k = __copyto!!(F_k, F_new)
         f_k = f_new
 
         # Store function value
         history_f_k[k % M + 1] = f_new
     end
-    return SciMLBase.build_solution(prob, alg, x, F_k; retcode = ReturnCode.MaxIters)
+
+    return build_solution(prob, alg, x, F_k; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index eb2ea1f5f..5cc7cdbbb 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -1,86 +1,85 @@
 """
-`Falsi`: A non-allocating regula falsi method
+    Falsi()
+
+A non-allocating regula falsi method
 """
 struct Falsi <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
-        maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-        kwargs...)
+        maxiters = 1000, abstol = nothing, kwargs...)
+    @assert !isinplace(prob) "`Falsi` only supports OOP problems."
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
+    abstol = _get_tolerance(abstol,
+        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+
     if iszero(fl)
-        return SciMLBase.build_solution(prob, alg, left, fl;
-            retcode = ReturnCode.ExactSolutionLeft, left = left,
-            right = right)
-    elseif iszero(fr)
-        return SciMLBase.build_solution(prob, alg, right, fr;
-            retcode = ReturnCode.ExactSolutionRight, left = left,
-            right = right)
+        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
+            left, right)
+    end
+
+    if iszero(fr)
+        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+            left, right)
     end
 
+    # Regula Falsi Steps
     i = 1
-    if !iszero(fr)
-        while i < maxiters
-            if nextfloat_tdir(left, prob.tspan...) == right
-                return SciMLBase.build_solution(prob, alg, left, fl;
-                    retcode = ReturnCode.FloatingPointLimit,
-                    left = left, right = right)
-            end
-            mid = (fr * left - fl * right) / (fr - fl)
-            for i in 1:10
-                mid = max_tdir(left, prevfloat_tdir(mid, prob.tspan...), prob.tspan...)
-            end
-            if mid == right || mid == left
-                break
-            end
-            fm = f(mid)
-            if abs((right - left) / 2) < abstol
-                return SciMLBase.build_solution(prob, alg, mid, fm;
-                    retcode = ReturnCode.Success,
-                    left = left, right = right)
-            end
-            if iszero(fm)
-                right = mid
-                break
-            end
-            if sign(fl) == sign(fm)
-                fl = fm
-                left = mid
-            else
-                fr = fm
-                right = mid
-            end
-            i += 1
+    while i < maxiters
+        if __nextfloat_tdir(left, prob.tspan...) == right
+            return build_solution(prob, alg, left, fl; left, right,
+                retcode = ReturnCode.FloatingPointLimit)
+        end
+
+        mid = (fr * left - fl * right) / (fr - fl)
+        for _ in 1:10
+            mid = __max_tdir(left, __prevfloat_tdir(mid, prob.tspan...), prob.tspan...)
         end
+
+        (mid == left || mid == right) && break
+
+        fm = f(mid)
+        if abs((right - left) / 2) < abstol
+            return build_solution(prob, alg, mid, fm; left, right,
+                retcode = ReturnCode.Success)
+        end
+
+        if abs(fm) < abstol
+            right = mid
+            break
+        end
+
+        if sign(fl) == sign(fm)
+            fl, left = fm, mid
+        else
+            fr, right = fm, mid
+        end
+        i += 1
     end
 
     while i < maxiters
         mid = (left + right) / 2
-        (mid == left || mid == right) &&
-            return SciMLBase.build_solution(prob, alg, left, fl;
-                retcode = ReturnCode.FloatingPointLimit,
-                left = left, right = right)
+        if (mid == left || mid == right)
+            return build_solution(prob, alg, left, fl; left, right,
+                retcode = ReturnCode.FloatingPointLimit)
+        end
+
         fm = f(mid)
-        if abs((right - left) / 2) < abstol
-            return SciMLBase.build_solution(prob, alg, mid, fm;
-                retcode = ReturnCode.Success,
-                left = left, right = right)
+        if abs((right - left) / 2) < abstol || abs(fm) < abstol
+            return build_solution(prob, alg, mid, fm; left, right,
+                retcode = ReturnCode.Success)
         end
-        if iszero(fm)
-            right = mid
-            fr = fm
-        elseif sign(fm) == sign(fl)
-            left = mid
-            fl = fm
+
+        if sign(fl * fm) < 0
+            right, fr = mid, fm
         else
-            right = mid
-            fr = fm
+            left, fl = mid, fm
         end
         i += 1
     end
 
     return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
-        left = left, right = right)
+        left, right)
 end
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/trustRegion.jl
index 0fba7b12f..d644f5fb7 100644
--- a/lib/SimpleNonlinearSolve/src/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/trustRegion.jl
@@ -1,58 +1,51 @@
 """
-```julia
-SimpleTrustRegion(; chunk_size = Val{0}(),
-                    autodiff = Val{true}(),
-                    diff_type = Val{:forward},
-                    max_trust_radius::Real = 0.0,
-                    initial_trust_radius::Real = 0.0,
-                    step_threshold::Real = 0.1,
-                    shrink_threshold::Real = 0.25,
-                    expand_threshold::Real = 0.75,
-                    shrink_factor::Real = 0.25,
-                    expand_factor::Real = 2.0,
-                    max_shrink_times::Int = 32
-```
+    SimpleTrustRegion(; chunk_size = Val{0}(), autodiff = Val{true}(),
+                        diff_type = Val{:forward}, max_trust_radius::Real = 0.0,
+                        initial_trust_radius::Real = 0.0, step_threshold::Real = 0.1,
+                        shrink_threshold::Real = 0.25, expand_threshold::Real = 0.75,
+                        shrink_factor::Real = 0.25, expand_factor::Real = 2.0,
+                        max_shrink_times::Int = 32)
 
 A low-overhead implementation of a trust-region solver.
 
 ### Keyword Arguments
 
-- `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
-  system. This allows for multiple derivative columns to be computed simultaneously,
-  improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
-  default chunk size mechanism. For more details, see the documentation for
-  [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
-- `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
-  Note that this argument is ignored if an analytical Jacobian is passed; as that will be
-  used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
-  If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
-- `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
-  `Val{:forward}` for forward finite differences. For more details on the choices, see the
-  [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
-- `max_trust_radius`: the maximum radius of the trust region. Defaults to
-  `max(norm(f(u0)), maximum(u0) - minimum(u0))`.
-- `initial_trust_radius`: the initial trust region radius. Defaults to
-  `max_trust_radius / 11`.
-- `step_threshold`: the threshold for taking a step. In every iteration, the threshold is
-  compared with a value `r`, which is the actual reduction in the objective function divided
-  by the predicted reduction. If `step_threshold > r` the model is not a good approximation,
-  and the step is rejected. Defaults to `0.1`. For more details, see
-  [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4)
-- `shrink_threshold`: the threshold for shrinking the trust region radius. In every
-  iteration, the threshold is compared with a value `r` which is the actual reduction in the
-  objective function divided by the predicted reduction. If `shrink_threshold > r` the trust
-  region radius is shrunk by `shrink_factor`. Defaults to `0.25`. For more details, see
-  [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4)
-- `expand_threshold`: the threshold for expanding the trust region radius. If a step is
-  taken, i.e `step_threshold < r` (with `r` defined in `shrink_threshold`), a check is also
-  made to see if `expand_threshold < r`. If that is true, the trust region radius is
-  expanded by `expand_factor`. Defaults to `0.75`.
-- `shrink_factor`: the factor to shrink the trust region radius with if
-  `shrink_threshold > r` (with `r` defined in `shrink_threshold`). Defaults to `0.25`.
-- `expand_factor`: the factor to expand the trust region radius with if
-  `expand_threshold < r` (with `r` defined in `shrink_threshold`). Defaults to `2.0`.
-- `max_shrink_times`: the maximum number of times to shrink the trust region radius in a
-  row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`.
+  - `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
+    system. This allows for multiple derivative columns to be computed simultaneously,
+    improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
+    default chunk size mechanism. For more details, see the documentation for
+    [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
+  - `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
+    Note that this argument is ignored if an analytical Jacobian is passed; as that will be
+    used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
+    If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
+  - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
+    `Val{:forward}` for forward finite differences. For more details on the choices, see the
+    [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+  - `max_trust_radius`: the maximum radius of the trust region. Defaults to
+    `max(norm(f(u0)), maximum(u0) - minimum(u0))`.
+  - `initial_trust_radius`: the initial trust region radius. Defaults to
+    `max_trust_radius / 11`.
+  - `step_threshold`: the threshold for taking a step. In every iteration, the threshold is
+    compared with a value `r`, which is the actual reduction in the objective function divided
+    by the predicted reduction. If `step_threshold > r` the model is not a good approximation,
+    and the step is rejected. Defaults to `0.1`. For more details, see
+    [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4)
+  - `shrink_threshold`: the threshold for shrinking the trust region radius. In every
+    iteration, the threshold is compared with a value `r` which is the actual reduction in the
+    objective function divided by the predicted reduction. If `shrink_threshold > r` the trust
+    region radius is shrunk by `shrink_factor`. Defaults to `0.25`. For more details, see
+    [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4)
+  - `expand_threshold`: the threshold for expanding the trust region radius. If a step is
+    taken, i.e `step_threshold < r` (with `r` defined in `shrink_threshold`), a check is also
+    made to see if `expand_threshold < r`. If that is true, the trust region radius is
+    expanded by `expand_factor`. Defaults to `0.75`.
+  - `shrink_factor`: the factor to shrink the trust region radius with if
+    `shrink_threshold > r` (with `r` defined in `shrink_threshold`). Defaults to `0.25`.
+  - `expand_factor`: the factor to expand the trust region radius with if
+    `expand_threshold < r` (with `r` defined in `shrink_threshold`). Defaults to `2.0`.
+  - `max_shrink_times`: the maximum number of times to shrink the trust region radius in a
+    row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`.
 """
 struct SimpleTrustRegion{T, CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
     max_trust_radius::T
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 228006409..df3374d86 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -5,22 +5,35 @@ function ForwardDiff.checktag(::Type{<:ForwardDiff.Tag{<:SimpleNonlinearSolveTag
     return true
 end
 
-# """
-#   prevfloat_tdir(x, x0, x1)
+"""
+    __prevfloat_tdir(x, x0, x1)
 
-# Move `x` one floating point towards x0.
-# """
-# function prevfloat_tdir(x, x0, x1)
-#     x1 > x0 ? prevfloat(x) : nextfloat(x)
-# end
+Move `x` one floating point towards x0.
+"""
+__prevfloat_tdir(x, x0, x1) = ifelse(x1 > x0, prevfloat(x), nextfloat(x))
 
-# function nextfloat_tdir(x, x0, x1)
-#     x1 > x0 ? nextfloat(x) : prevfloat(x)
-# end
+"""
+    __nextfloat_tdir(x, x0, x1)
 
-# function max_tdir(a, b, x0, x1)
-#     x1 > x0 ? max(a, b) : min(a, b)
-# end
+Move `x` one floating point towards x1.
+"""
+__nextfloat_tdir(x, x0, x1) = ifelse(x1 > x0, nextfloat(x), prevfloat(x))
+
+"""
+    __max_tdir(a, b, x0, x1)
+
+Return the maximum of `a` and `b` if `x1 > x0`, otherwise return the minimum.
+"""
+__max_tdir(a, b, x0, x1) = ifelse(x1 > x0, max(a, b), min(a, b))
+
+__cvt_real(::Type{T}, ::Nothing) where {T} = nothing
+__cvt_real(::Type{T}, x) where {T} = real(T(x))
+
+_get_tolerance(η, ::Type{T}) where {T} = __cvt_real(T, η)
+function _get_tolerance(::Nothing, ::Type{T}) where {T}
+    η = real(oneunit(T)) * (eps(real(one(T))))^(4 // 5)
+    return _get_tolerance(η, T)
+end
 
 __standard_tag(::Nothing, x) = ForwardDiff.Tag(SimpleNonlinearSolveTag(), eltype(x))
 __standard_tag(tag::ForwardDiff.Tag, _) = tag

From 408243e1456b0e95bbb92cfd3dbf09bb57121743 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 23 Nov 2023 02:43:44 -0500
Subject: [PATCH 223/375] Use a macro for compile time compatibility between
 inplace and oop versions

---
 .../src/SimpleNonlinearSolve.jl               |  13 +-
 lib/SimpleNonlinearSolve/src/bisection.jl     |  50 +++-
 lib/SimpleNonlinearSolve/src/broyden.jl       |  43 ++--
 lib/SimpleNonlinearSolve/src/dfsane.jl        | 224 +++++++++---------
 lib/SimpleNonlinearSolve/src/falsi.jl         |  77 +++---
 lib/SimpleNonlinearSolve/src/klement.jl       |  67 ++++--
 lib/SimpleNonlinearSolve/src/raphson.jl       |  10 +-
 .../src/rewrite_inplace.jl                    | 161 +++++++++++++
 lib/SimpleNonlinearSolve/src/ridder.jl        |  71 +++---
 lib/SimpleNonlinearSolve/src/utils.jl         |  91 +------
 10 files changed, 457 insertions(+), 350 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/rewrite_inplace.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index e0293ee6c..208e0e15f 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -25,6 +25,7 @@ abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorit
 abstract type AbstractNewtonAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 
 include("utils.jl")
+include("rewrite_inplace.jl")
 
 # Nonlinear Solvera
 include("raphson.jl")
@@ -33,12 +34,12 @@ include("broyden.jl")
 include("klement.jl")
 # include("trustRegion.jl")
 # include("halley.jl")
-include("dfsane.jl")
+# include("dfsane.jl")
 
 # Interval Nonlinear Solvers
 include("bisection.jl")
 include("falsi.jl")
-# include("ridder.jl")
+include("ridder.jl")
 # include("brent.jl")
 # include("alefeld.jl")
 # include("itp.jl")
@@ -88,9 +89,9 @@ include("falsi.jl")
 #     end
 # end
 
-export SimpleBroyden, SimpleDFSane, SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson
-export Bisection, Falsi
-# export Bisection, Brent, LBroyden, SimpleHalley,
-#     Ridder, SimpleTrustRegion, Alefeld, ITP
+export SimpleBroyden, SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson
+# SimpleDFSane, SimpleTrustRegion, SimpleHalley
+export Bisection, Falsi, Ridder
+# export , Brent, LBroyden, Alefeld, ITP
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bisection.jl
index 9b1394bc6..42bb2cad0 100644
--- a/lib/SimpleNonlinearSolve/src/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bisection.jl
@@ -39,17 +39,57 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
             left, right)
     end
 
-    for _ in 1:maxiters
+    i = 1
+    if !iszero(fr)
+        while i < maxiters
+            mid = (left + right) / 2
+            (mid == left || mid == right) &&
+                return build_solution(prob, alg, left, fl; left, right,
+                    retcode = ReturnCode.FloatingPointLimit)
+            fm = f(mid)
+            if abs((right - left) / 2) < abstol
+                return build_solution(prob, alg, mid, fm; retcode = ReturnCode.Success,
+                    left, right)
+            end
+            if iszero(fm)
+                right = mid
+                break
+            end
+            if sign(fl) == sign(fm)
+                fl = fm
+                left = mid
+            else
+                fr = fm
+                right = mid
+            end
+            i += 1
+        end
+    end
+
+    sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
+        maxiters = maxiters - i, prob, alg)
+
+    sol !== nothing && return sol
+
+    return build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
+end
+
+function __bisection(left, right, fl, fr, f::F; abstol, maxiters, prob, alg) where {F}
+    i = 1
+    sol = nothing
+    while i < maxiters
         mid = (left + right) / 2
         if (mid == left || mid == right)
-            return build_solution(prob, alg, left, fl; left, right,
+            sol = build_solution(prob, alg, left, fl; left, right,
                 retcode = ReturnCode.FloatingPointLimit)
+            break
         end
 
         fm = f(mid)
         if abs((right - left) / 2) < abstol || abs(fm) < abstol
-            return build_solution(prob, alg, mid, fm; left, right,
+            sol = build_solution(prob, alg, mid, fm; left, right,
                 retcode = ReturnCode.Success)
+            break
         end
 
         if sign(fl * fm) < 0
@@ -57,7 +97,9 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
         else
             left, fl = mid, fm
         end
+
+        i += 1
     end
 
-    return build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
+    return sol, i, left, right, fl, fr
 end
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
index 7587168a4..aaf959cb5 100644
--- a/lib/SimpleNonlinearSolve/src/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -9,44 +9,45 @@ struct SimpleBroyden <: AbstractSimpleNonlinearSolveAlgorithm end
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         termination_condition = nothing, kwargs...)
-    f = isinplace(prob) ? (du, u) -> prob.f(du, u, prob.p) : u -> prob.f(u, prob.p)
-    x = float(prob.u0)
+    @bb x = copy(float(prob.u0))
     fx = _get_fx(prob, x)
-    xo, δx, fprev, δf = __copy(x), __copy(x), __copy(fx), __copy(fx)
+
+    @bb xo = copy(x)
+    @bb δx = copy(x)
+    @bb δf = copy(fx)
+    @bb fprev = copy(fx)
 
     J⁻¹ = __init_identity_jacobian(fx, x)
-    J⁻¹δf, xᵀJ⁻¹ = __copy(x), __copy(x)
-    δJ⁻¹, δJ⁻¹n = __copy(x, J⁻¹), __copy(x)
+    @bb J⁻¹δf = copy(x)
+    @bb xᵀJ⁻¹ = copy(x)
+    @bb δJ⁻¹n = copy(x)
+    @bb δJ⁻¹ = copy(J⁻¹)
 
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
 
     for _ in 1:maxiters
-        δx = _restructure(δx, __mul!!(_vec(δx), J⁻¹, _vec(fprev)))
-        x = __sub!!(x, xo, δx)
-        fx = __eval_f(prob, f, fx, x)
-        δf = __sub!!(δf, fx, fprev)
+        @bb δx = J⁻¹ × vec(fprev)
+        @bb @. x = xo - δx
+        fx = __eval_f(prob, fx, x)
+        @bb @. δf = fx - fprev
 
         # Termination Checks
         tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
         tc_sol !== nothing && return tc_sol
 
-        J⁻¹δf = _restructure(J⁻¹δf, __mul!!(_vec(J⁻¹δf), J⁻¹, _vec(δf)))
-        δx = __neg!!(δx)
+        @bb J⁻¹δf = J⁻¹ × vec(δf)
+        @bb δx .*= -1
         d = dot(δx, J⁻¹δf)
-        xᵀJ⁻¹ = _restructure(xᵀJ⁻¹, __mul!!(_vec(xᵀJ⁻¹), _vec(δx)', J⁻¹))
+        @bb xᵀJ⁻¹ = transpose(J⁻¹) × vec(δx)
 
-        if ArrayInterface.can_setindex(δJ⁻¹n)
-            @. δJ⁻¹n = (δx - J⁻¹δf) / d
-        else
-            δJ⁻¹n = @. (δx - J⁻¹δf) / d
-        end
+        @bb @. δJ⁻¹n = (δx - J⁻¹δf) / d
 
-        δJ⁻¹ = __mul!!(δJ⁻¹, δJ⁻¹n, xᵀJ⁻¹')
-        J⁻¹ = __add!!(J⁻¹, δJ⁻¹)
+        @bb δJ⁻¹ = δJ⁻¹n × transpose(xᵀJ⁻¹)
+        @bb J⁻¹ .+= δJ⁻¹
 
-        xo = __copyto!!(xo, x)
-        fprev = __copyto!!(fprev, fx)
+        @bb copyto!(xo, x)
+        @bb copyto!(fprev, fx)
     end
 
     return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
index d646171d5..0ecc545f6 100644
--- a/lib/SimpleNonlinearSolve/src/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -54,116 +54,116 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         termination_condition = nothing, kwargs...)
 
-    f = isinplace(prob) ? (du, u) -> prob.f(du, u, prob.p) : u -> prob.f(u, prob.p)
-
-    x = float(prob.u0)
-    fx = _get_fx(prob, x)
-    T = eltype(x)
-
-    σ_min = T(alg.σ_min)
-    σ_max = T(alg.σ_max)
-    σ_k = T(alg.σ_1)
-
-    M = alg.M
-    γ = T(alg.γ)
-    τ_min = T(alg.τ_min)
-    τ_max = T(alg.τ_max)
-    nexp = alg.nexp
-    η_strategy = alg.η_strategy
-
-    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
-        termination_condition)
-
-    ff = if isinplace(prob)
-        function (_fx, x)
-            f(_fx, x)
-            f_k = norm(_fx)^nexp
-            return f_k, _fx
-        end
-    else
-        function (x)
-            _fx = f(x)
-            f_k = norm(_fx)^nexp
-            return f_k, _fx
-        end
-    end
-
-    generate_history(f_k, M) = fill(f_k, M)
-
-    f_k, F_k = isinplace(prob) ? ff(fx, x) : ff(x)
-    F_k = __copy(F_k)
-    α_1 = one(T)
-    f_1 = f_k
-    history_f_k = generate_history(f_k, M)
-
-    # Generate the cache
-    d, xo, x_cache, δx, δf = __copy(x), __copy(x), __copy(x), __copy(x), __copy(x)
-    α_tp, α_tm = __copy(x), __copy(x)
-
-    for k in 1:maxiters
-        # Spectral parameter range check
-        σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
-
-        # Line search direction
-        d = __broadcast!!(d, *, -σ_k, F_k)
-
-        η = η_strategy(f_1, k, x, F_k)
-        f̄ = maximum(history_f_k)
-        α_p = α_1
-        α_m = α_1
-
-        x_cache = __broadcast!!(x_cache, *, α_p, d)
-        x = __broadcast!!(x, +, x_cache)
-
-        f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
-
-        # FIXME: This part is not correctly implemented
-        while true
-            criteria = f̄ + η - γ * α_p^2 * f_k
-            f_new ≤ criteria && break
-
-            if ArrayInterface.can_setindex(α_tp) && !(x isa Number)
-                @. α_tp = α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
-            else
-                α_tp = @. α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
-            end
-            x_cache = __broadcast!!(x_cache, *, α_m, d)
-            x = __broadcast!!(x, -, x_cache)
-            f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
-
-            f_new ≤ criteria && break
-
-            if ArrayInterface.can_setindex(α_tm) && !(x isa Number)
-                @. α_tm = α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
-                @. α_p = clamp(α_tp, τ_min * α_p, τ_max * α_p)
-                @. α_m = clamp(α_tm, τ_min * α_m, τ_max * α_m)
-            else
-                α_tm = @. α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
-                α_p = @. clamp(α_tp, τ_min * α_p, τ_max * α_p)
-                α_m = @. clamp(α_tm, τ_min * α_m, τ_max * α_m)
-            end
-            x_cache = __broadcast!!(x_cache, *, α_p, d)
-            x = __broadcast!!(x, +, x_cache)
-            f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
-        end
-
-        tc_sol = check_termination(tc_cache, f_new, x, xo, prob, alg)
-        tc_sol !== nothing && return tc_sol
-
-        # Update spectral parameter
-        δx = __broadcast!!(δx, -, x, xo)
-        δf = __broadcast!!(δf, -, F_new, F_k)
-
-        σ_k = dot(δx, δx) / dot(δx, δf)
-
-        # Take step
-        xo = __copyto!!(xo, x)
-        F_k = __copyto!!(F_k, F_new)
-        f_k = f_new
-
-        # Store function value
-        history_f_k[k % M + 1] = f_new
-    end
-
-    return build_solution(prob, alg, x, F_k; retcode = ReturnCode.MaxIters)
+    # f = isinplace(prob) ? (du, u) -> prob.f(du, u, prob.p) : u -> prob.f(u, prob.p)
+
+    # x = float(prob.u0)
+    # fx = _get_fx(prob, x)
+    # T = eltype(x)
+
+    # σ_min = T(alg.σ_min)
+    # σ_max = T(alg.σ_max)
+    # σ_k = T(alg.σ_1)
+
+    # M = alg.M
+    # γ = T(alg.γ)
+    # τ_min = T(alg.τ_min)
+    # τ_max = T(alg.τ_max)
+    # nexp = alg.nexp
+    # η_strategy = alg.η_strategy
+
+    # abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+    #     termination_condition)
+
+    # ff = if isinplace(prob)
+    #     function (_fx, x)
+    #         f(_fx, x)
+    #         f_k = norm(_fx)^nexp
+    #         return f_k, _fx
+    #     end
+    # else
+    #     function (x)
+    #         _fx = f(x)
+    #         f_k = norm(_fx)^nexp
+    #         return f_k, _fx
+    #     end
+    # end
+
+    # generate_history(f_k, M) = fill(f_k, M)
+
+    # f_k, F_k = isinplace(prob) ? ff(fx, x) : ff(x)
+    # F_k = __copy(F_k)
+    # α_1 = one(T)
+    # f_1 = f_k
+    # history_f_k = generate_history(f_k, M)
+
+    # # Generate the cache
+    # d, xo, x_cache, δx, δf = __copy(x), __copy(x), __copy(x), __copy(x), __copy(x)
+    # α_tp, α_tm = __copy(x), __copy(x)
+
+    # for k in 1:maxiters
+    #     # Spectral parameter range check
+    #     σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
+
+    #     # Line search direction
+    #     d = __broadcast!!(d, *, -σ_k, F_k)
+
+    #     η = η_strategy(f_1, k, x, F_k)
+    #     f̄ = maximum(history_f_k)
+    #     α_p = α_1
+    #     α_m = α_1
+
+    #     x_cache = __broadcast!!(x_cache, *, α_p, d)
+    #     x = __broadcast!!(x, +, x_cache)
+
+    #     f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
+
+    #     # FIXME: This part is not correctly implemented
+    #     while true
+    #         criteria = f̄ + η - γ * α_p^2 * f_k
+    #         f_new ≤ criteria && break
+
+    #         if ArrayInterface.can_setindex(α_tp) && !(x isa Number)
+    #             @. α_tp = α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
+    #         else
+    #             α_tp = @. α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
+    #         end
+    #         x_cache = __broadcast!!(x_cache, *, α_m, d)
+    #         x = __broadcast!!(x, -, x_cache)
+    #         f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
+
+    #         f_new ≤ criteria && break
+
+    #         if ArrayInterface.can_setindex(α_tm) && !(x isa Number)
+    #             @. α_tm = α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
+    #             @. α_p = clamp(α_tp, τ_min * α_p, τ_max * α_p)
+    #             @. α_m = clamp(α_tm, τ_min * α_m, τ_max * α_m)
+    #         else
+    #             α_tm = @. α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
+    #             α_p = @. clamp(α_tp, τ_min * α_p, τ_max * α_p)
+    #             α_m = @. clamp(α_tm, τ_min * α_m, τ_max * α_m)
+    #         end
+    #         x_cache = __broadcast!!(x_cache, *, α_p, d)
+    #         x = __broadcast!!(x, +, x_cache)
+    #         f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
+    #     end
+
+    #     tc_sol = check_termination(tc_cache, f_new, x, xo, prob, alg)
+    #     tc_sol !== nothing && return tc_sol
+
+    #     # Update spectral parameter
+    #     δx = __broadcast!!(δx, -, x, xo)
+    #     δf = __broadcast!!(δf, -, F_new, F_k)
+
+    #     σ_k = dot(δx, δx) / dot(δx, δf)
+
+    #     # Take step
+    #     xo = __copyto!!(xo, x)
+    #     F_k = __copyto!!(F_k, F_new)
+    #     f_k = f_new
+
+    #     # Store function value
+    #     history_f_k[k % M + 1] = f_new
+    # end
+
+    # return build_solution(prob, alg, x, F_k; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/falsi.jl
index 5cc7cdbbb..9db7d6cf1 100644
--- a/lib/SimpleNonlinearSolve/src/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/falsi.jl
@@ -1,7 +1,7 @@
 """
     Falsi()
 
-A non-allocating regula falsi method
+A non-allocating regula falsi method.
 """
 struct Falsi <: AbstractBracketingAlgorithm end
 
@@ -26,59 +26,44 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
     end
 
     # Regula Falsi Steps
-    i = 1
-    while i < maxiters
-        if __nextfloat_tdir(left, prob.tspan...) == right
-            return build_solution(prob, alg, left, fl; left, right,
-                retcode = ReturnCode.FloatingPointLimit)
-        end
+    i = 0
+    if !iszero(fr)
+        while i < maxiters
+            if __nextfloat_tdir(left, prob.tspan...) == right
+                return build_solution(prob, alg, left, fl; left, right,
+                    retcode = ReturnCode.FloatingPointLimit)
+            end
 
-        mid = (fr * left - fl * right) / (fr - fl)
-        for _ in 1:10
-            mid = __max_tdir(left, __prevfloat_tdir(mid, prob.tspan...), prob.tspan...)
-        end
+            mid = (fr * left - fl * right) / (fr - fl)
+            for _ in 1:10
+                mid = __max_tdir(left, __prevfloat_tdir(mid, prob.tspan...), prob.tspan...)
+            end
 
-        (mid == left || mid == right) && break
+            (mid == left || mid == right) && break
 
-        fm = f(mid)
-        if abs((right - left) / 2) < abstol
-            return build_solution(prob, alg, mid, fm; left, right,
-                retcode = ReturnCode.Success)
-        end
+            fm = f(mid)
+            if abs((right - left) / 2) < abstol
+                return build_solution(prob, alg, mid, fm; left, right,
+                    retcode = ReturnCode.Success)
+            end
 
-        if abs(fm) < abstol
-            right = mid
-            break
-        end
+            if abs(fm) < abstol
+                right = mid
+                break
+            end
 
-        if sign(fl) == sign(fm)
-            fl, left = fm, mid
-        else
-            fr, right = fm, mid
+            if sign(fl) == sign(fm)
+                fl, left = fm, mid
+            else
+                fr, right = fm, mid
+            end
+            i += 1
         end
-        i += 1
     end
 
-    while i < maxiters
-        mid = (left + right) / 2
-        if (mid == left || mid == right)
-            return build_solution(prob, alg, left, fl; left, right,
-                retcode = ReturnCode.FloatingPointLimit)
-        end
-
-        fm = f(mid)
-        if abs((right - left) / 2) < abstol || abs(fm) < abstol
-            return build_solution(prob, alg, mid, fm; left, right,
-                retcode = ReturnCode.Success)
-        end
-
-        if sign(fl * fm) < 0
-            right, fr = mid, fm
-        else
-            left, fl = mid, fm
-        end
-        i += 1
-    end
+    sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
+        maxiters = maxiters - i, prob, alg)
+    sol !== nothing && return sol
 
     return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
         left, right)
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
index 3d22d1c07..7b9a878ad 100644
--- a/lib/SimpleNonlinearSolve/src/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -1,14 +1,14 @@
 """
     SimpleKlement()
 
-A low-overhead implementation of [Klement](https://jatm.com.br/jatm/article/view/373).
+A low-overhead implementation of [Klement](https://jatm.com.br/jatm/article/view/373). This
+method is non-allocating on scalar and static array problems.
 """
 struct SimpleKlement <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         termination_condition = nothing, kwargs...)
-    f = isinplace(prob) ? (du, u) -> prob.f(du, u, prob.p) : u -> prob.f(u, prob.p)
     x = float(prob.u0)
     T = eltype(x)
     fx = _get_fx(prob, x)
@@ -18,51 +18,68 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
 
-    δx, fprev, xo, δf, d = __copy(fx), __copy(fx), __copy(x), __copy(fx), __copy(x)
+    @bb δx = copy(x)
+    @bb fprev = copy(fx)
+    @bb xo = copy(x)
+    @bb δf = copy(fx)
+    @bb d = copy(x)
+
     J = __init_identity_jacobian(fx, x)
-    J_cache, δx² = __copy(J), __copy(x)
+    @bb J_cache = copy(J)
+    @bb δx² = copy(x)
+    @bb J_cache2 = copy(J)
+    @bb F = copy(J)
 
     for _ in 1:maxiters
         if x isa Number
             J < singular_tol && (J = __init_identity_jacobian!!(J))
-            F = J
+            F_ = J
         else
-            F = lu(J; check = false)
+            @bb copyto!(F, J)
+            if setindex_trait(F) === CanSetindex()
+                F_ = lu!(F; check = false)
+            else
+                F_ = lu(F; check = false)
+            end
 
             # Singularity test
-            if any(x -> abs(x) < singular_tol, @view(F.U[diagind(F.U)]))
+            if !issuccess(F_)
                 J = __init_identity_jacobian!!(J)
-                F = lu(J; check = false)
+                if setindex_trait(J) === CanSetindex()
+                    lu!(J; check = false)
+                else
+                    J = lu(J; check = false)
+                end
             end
         end
 
-        δx = __copyto!!(δx, fprev)
-        δx = __ldiv!!(F, δx)
-        x = __sub!!(x, xo, δx)
-        fx = __eval_f(prob, f, fx, x)
+        @bb copyto!(δx, fprev)
+        δx = __ldiv!!(F_, δx)
+        @bb @. x = xo - δx
+        fx = __eval_f(prob, fx, x)
 
         # Termination Checks
         tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
         tc_sol !== nothing && return tc_sol
 
-        δx = __neg!!(δx)
-        δf = __sub!!(δf, fx, fprev)
+        @bb δx .*= -1
+        @bb @. δf = fx - fprev
 
         # Prevent division by 0
-        δx² = __broadcast!!(δx², abs2, δx)
-        J_cache = __broadcast!!(J_cache, abs2, J)
-        d = _restructure(d, __mul!!(_vec(d), J_cache', _vec(δx²)))
-        d = __broadcast!!(d, Base.Fix2(max, singular_tol), d)
+        @bb @. δx² = δx^2
+        @bb @. J_cache = J^2
+        @bb d = transpose(J_cache) × vec(δx²)
+        @bb @. d = max(d, singular_tol)
 
-        δx² = _restructure(δx², __mul!!(_vec(δx²), J, _vec(δx)))
-        δf = __sub!!(δf, δx²)
-        δf = __broadcast!!(δf, /, δf, d)
+        @bb δx² = J × vec(δx)
+        @bb @. δf = (δf - δx²) / d
 
-        J_cache = __mul!!(J_cache, _vec(δf), _vec(δx)')
-        J_cache = __broadcast!!(J_cache, *, J_cache, J)
-        J_cache = __mul!!(J_cache, J_cache, J)
+        _vδf, _vδx = vec(δf), vec(δx)
+        @bb J_cache = _vδf × transpose(_vδx)
+        @bb @. J_cache *= J
+        @bb J_cache2 = J_cache × J
 
-        J = __add!!(J, J_cache)
+        @bb @. J += J_cache2
     end
 
     return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
index a1974ba88..1b63656de 100644
--- a/lib/SimpleNonlinearSolve/src/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -9,7 +9,7 @@ and static array problems.
 
     As part of the decreased overhead, this method omits some of the higher level error
     catching of the other methods. Thus, to see better error messages, use one of the other
-    methods like `NewtonRaphson`
+    methods like `NewtonRaphson`.
 
 ### Keyword Arguments
 
@@ -27,9 +27,9 @@ const SimpleGaussNewton = SimpleNewtonRaphson
 function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
         alg::SimpleNewtonRaphson, args...; abstol = nothing, reltol = nothing,
         maxiters = 1000, termination_condition = nothing, kwargs...)
-    x = float(prob.u0)
+    @bb x = copy(float(prob.u0))
     fx = _get_fx(prob, x)
-    xo = __copy(x)
+    @bb xo = copy(x)
     J, jac_cache = jacobian_cache(alg.ad, prob.f, fx, x, prob.p)
 
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
@@ -48,9 +48,9 @@ function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresPr
             tc_sol !== nothing && return tc_sol
         end
 
-        xo = __copyto!!(xo, x)
+        @bb copyto!(xo, x)
         Δx = _restructure(x, dfx \ _vec(fx))
-        x = __sub!!(x, Δx)
+        @bb x .-= Δx
     end
 
     return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
diff --git a/lib/SimpleNonlinearSolve/src/rewrite_inplace.jl b/lib/SimpleNonlinearSolve/src/rewrite_inplace.jl
new file mode 100644
index 000000000..f0d80af23
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/rewrite_inplace.jl
@@ -0,0 +1,161 @@
+# Take a inplace code and rewrite it to be maybe-inplace
+# I will take this code out into a separate package because this is useful even in
+# NonlinearSolve.jl
+function __bangbang(M, expr; depth = 1)
+    new_expr = nothing
+    if expr.head == :call
+        @assert length(expr.args)≥2 "Expected a function call with atleast 1 argument. \
+                                     Got `$(expr)`."
+        f, a, args... = expr.args
+        g = get(OP_MAPPING, f, nothing)
+        if f == :copy && length(args) == 0
+            # Special case for copy with single argument
+            new_expr = :($(g)($(setindex_trait)($(a)), $(a)))
+        elseif g !== nothing
+            new_expr = :($(a) = $(g)($(setindex_trait)($(a)), $(a), $(args...)))
+        end
+    elseif expr.head == :(=)
+        a, rhs_expr = expr.args
+        if rhs_expr.head == :call
+            f, b, args... = rhs_expr.args
+            g = get(OP_MAPPING, f, nothing)
+            if g !== nothing
+                new_expr = :($(a) = $(g)($(setindex_trait)($(b)), $(b), $(args...)))
+            elseif f == :×
+                @debug "Custom operator `×` detected in `$(expr)`."
+                c, args... = args
+                @assert length(args)==0 "Expected `×` to have only 2 arguments. \
+                                        Got `$(expr)`."
+                is_b_vec = b isa Expr && b.head == :call && b.args[1] == :vec
+                is_c_vec = c isa Expr && c.head == :call && c.args[1] == :vec
+                a_sym = gensym("a")
+                if is_b_vec
+                    if is_c_vec
+                        error("2 `vec`s detected with `×` in `$(expr)`. Use `dot` instead.")
+                    else
+                        new_expr = quote
+                            if $(setindex_trait)($(a)) === CanSetindex()
+                                $(a_sym) = $(_vec)($a)
+                                mul!($(a_sym), $(_vec)($(b.args[2])), $(c))
+                                $(a) = $(_restructure)($a, $(a_sym))
+                            else
+                                $(a) = $(_restructure)($a, $(_vec)($(b.args[2])) * $(c))
+                            end
+                        end
+                    end
+                else
+                    if is_c_vec
+                        new_expr = quote
+                            if $(setindex_trait)($(a)) === CanSetindex()
+                                $(a_sym) = $(_vec)($a)
+                                mul!($(a), $(b), $(_vec)($(c.args[2])))
+                                $(a) = $(_restructure)($a, $(a_sym))
+                            else
+                                $(a) = $(_restructure)($a, $(b) * $(_vec)($(c.args[2])))
+                            end
+                        end
+                    else
+                        new_expr = quote
+                            if $(setindex_trait)($(a)) === CanSetindex()
+                                mul!($(a), $(b), $(c))
+                            else
+                                $(a) = $(b) * $(c)
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    elseif expr.head == :(.=)
+        a, rhs_expr = expr.args
+        if rhs_expr isa Expr && rhs_expr.head == :(.)
+            f, arg_expr = rhs_expr.args
+            # f_broadcast = :(Base.Broadcast.BroadcastFunction($(f)))
+            new_expr = quote
+                if $(setindex_trait)($(a)) === CanSetindex()
+                    broadcast!($(f), $(a), $(arg_expr)...)
+                else
+                    $(a) = broadcast($(f), $(arg_expr)...)
+                end
+            end
+        end
+    elseif expr.head == :macrocall
+        # For @__dot__ there is a easier alternative
+        if expr.args[1] == Symbol("@__dot__")
+            main_expr = last(expr.args)
+            if main_expr isa Expr && main_expr.head == :(=)
+                a, rhs_expr = main_expr.args
+                new_expr = quote
+                    if $(setindex_trait)($(a)) === CanSetindex()
+                        @. $(main_expr)
+                    else
+                        $(a) = @. $(rhs_expr)
+                    end
+                end
+            end
+        end
+        if new_expr === nothing
+            new_expr = __bangbang(M, Base.macroexpand(M, expr; recursive = true);
+                depth = depth + 1)
+        end
+    else
+        f = expr.head # Things like :.-=, etc.
+        a, args... = expr.args
+        g = get(OP_MAPPING, f, nothing)
+        if g !== nothing
+            new_expr = :($(a) = $(g)($(setindex_trait)($(a)), $(a), $(args...)))
+        end
+    end
+    if new_expr !== nothing
+        if depth == 1
+            @debug "Replacing `$(expr)` with `$(new_expr)`"
+            return esc(new_expr)
+        else
+            return new_expr
+        end
+    end
+    error("`$(expr)` cannot be handled. Check the documentation for allowed expressions.")
+end
+
+macro bangbang(expr)
+    return __bangbang(__module__, expr)
+end
+
+# `bb` is the short form of bang-bang
+macro bb(expr)
+    return __bangbang(__module__, expr)
+end
+
+# Is Mutable or Not?
+abstract type AbstractMaybeSetindex end
+struct CannotSetindex <: AbstractMaybeSetindex end
+struct CanSetindex <: AbstractMaybeSetindex end
+
+# Common types should overload this via extensions, else it butchers type-inference
+setindex_trait(::Union{Number, SArray}) = CannotSetindex()
+setindex_trait(::Union{MArray, Array}) = CanSetindex()
+setindex_trait(A) = ifelse(ArrayInterface.can_setindex(A), CanSetindex(), CannotSetindex())
+
+# Operations
+const OP_MAPPING = Dict{Symbol, Symbol}(:copyto! => :__copyto!!,
+    :.-= => :__sub!!,
+    :.+= => :__add!!,
+    :.*= => :__mul!!,
+    :./= => :__div!!,
+    :copy => :__copy)
+
+@inline __copyto!!(::CannotSetindex, x, y) = y
+@inline __copyto!!(::CanSetindex, x, y) = (copyto!(x, y); x)
+
+@inline __broadcast!!(::CannotSetindex, op, x, args...) = broadcast(op, args...)
+@inline __broadcast!!(::CanSetindex, op, x, args...) = (broadcast!(op, x, args...); x)
+
+@inline __sub!!(S, x, args...) = __broadcast!!(S, -, x, x, args...)
+@inline __add!!(S, x, args...) = __broadcast!!(S, +, x, x, args...)
+@inline __mul!!(S, x, args...) = __broadcast!!(S, *, x, x, args...)
+@inline __div!!(S, x, args...) = __broadcast!!(S, /, x, x, args...)
+
+@inline __copy(::CannotSetindex, x) = x
+@inline __copy(::CanSetindex, x) = copy(x)
+@inline __copy(::CannotSetindex, x, y) = y
+@inline __copy(::CanSetindex, x, y) = copy(y)
diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index 41b43200e..0bed8ee75 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -1,25 +1,28 @@
 """
-`Ridder()`
+    Ridder()
 
-A non-allocating ridder method
+A non-allocating ridder method.
 """
 struct Ridder <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
-        maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-        kwargs...)
+        maxiters = 1000, abstol = nothing, kwargs...)
+    @assert !isinplace(prob) "`Ridder` only supports OOP problems."
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
+    abstol = _get_tolerance(abstol,
+        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+
     if iszero(fl)
-        return SciMLBase.build_solution(prob, alg, left, fl;
-            retcode = ReturnCode.ExactSolutionLeft, left = left,
-            right = right)
-    elseif iszero(fr)
-        return SciMLBase.build_solution(prob, alg, right, fr;
-            retcode = ReturnCode.ExactSolutionRight, left = left,
-            right = right)
+        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
+            left, right)
+    end
+
+    if iszero(fr)
+        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+            left, right)
     end
 
     xo = oftype(left, Inf)
@@ -27,23 +30,21 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
     if !iszero(fr)
         while i < maxiters
             mid = (left + right) / 2
-            (mid == left || mid == right) &&
-                return SciMLBase.build_solution(prob, alg, left, fl;
-                    retcode = ReturnCode.FloatingPointLimit,
-                    left = left, right = right)
+            if (mid == left || mid == right)
+                return build_solution(prob, alg, left, fl; left, right,
+                    retcode = ReturnCode.FloatingPointLimit)
             fm = f(mid)
             s = sqrt(fm^2 - fl * fr)
-            iszero(s) &&
-                return SciMLBase.build_solution(prob, alg, left, fl;
-                    retcode = ReturnCode.Failure,
-                    left = left, right = right)
+            if iszero(s)
+                return build_solution(prob, alg, left, fl; left, right,
+                    retcode = ReturnCode.Failure)
+            end
             x = mid + (mid - left) * sign(fl - fr) * fm / s
             fx = f(x)
             xo = x
             if abs((right - left) / 2) < abstol
-                return SciMLBase.build_solution(prob, alg, mid, fm;
-                    retcode = ReturnCode.Success,
-                    left = left, right = right)
+                return build_solution(prob, alg, mid, fm; retcode = ReturnCode.Success,
+                    left, right)
             end
             if iszero(fx)
                 right = x
@@ -67,28 +68,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
         end
     end
 
-    while i < maxiters
-        mid = (left + right) / 2
-        (mid == left || mid == right) &&
-            return SciMLBase.build_solution(prob, alg, left, fl;
-                retcode = ReturnCode.FloatingPointLimit,
-                left = left, right = right)
-        fm = f(mid)
-        if abs((right - left) / 2) < abstol
-            return SciMLBase.build_solution(prob, alg, mid, fm;
-                retcode = ReturnCode.Success,
-                left = left, right = right)
-        end
-        if iszero(fm)
-            right = mid
-            fr = fm
-        else
-            left = mid
-            fl = fm
-        end
-        i += 1
-    end
+    sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
+        maxiters = maxiters - i, prob, alg)
+    sol !== nothing && return sol
 
     return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
-        left = left, right = right)
+        left, right)
 end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index df3374d86..11caa7073 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -134,7 +134,7 @@ function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
         if DiffEqBase.has_jac(f)
             return nothing, nothing
         elseif ad isa AutoForwardDiff
-            J = ArrayInterface.can_setindex(x) ? similar(y, length(fx), length(x)) : nothing
+            J = ArrayInterface.can_setindex(x) ? similar(y, length(y), length(x)) : nothing
             return J, __get_jacobian_config(ad, _f, x)
         elseif ad isa AutoFiniteDiff
             return nothing, FiniteDiff.JacobianCache(copy(x), copy(y), copy(y), ad.fdtype)
@@ -150,6 +150,7 @@ __init_identity_jacobian(u::Number, _) = one(u)
 __init_identity_jacobian!!(J::Number) = one(J)
 function __init_identity_jacobian(u, fu)
     J = similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u))
+    fill!(J, zero(eltype(J)))
     J[diagind(J)] .= one(eltype(J))
     return J
 end
@@ -281,89 +282,5 @@ function check_termination(tc_cache, fx, x, xo, prob, alg,
     return nothing
 end
 
-# MaybeInplace
-@inline __copyto!!(::Number, x) = x
-@inline __copyto!!(::SArray, x) = x
-@inline __copyto!!(y::Union{MArray, Array}, x) = copyto!(y, x)
-@inline function __copyto!!(y::AbstractArray, x)
-    ArrayInterface.can_setindex(y) && return copyto!(y, x)
-    return x
-end
-
-@inline __sub!!(x::Number, Δx) = x - Δx
-@inline __sub!!(x::SArray, Δx) = x .- Δx
-@inline __sub!!(x::Union{MArray, Array}, Δx) = (x .-= Δx)
-@inline function __sub!!(x::AbstractArray, Δx)
-    ArrayInterface.can_setindex(x) && return (x .-= Δx)
-    return x .- Δx
-end
-
-@inline __sub!!(::Number, x, Δx) = x - Δx
-@inline __sub!!(::SArray, x, Δx) = x .- Δx
-@inline __sub!!(y::Union{MArray, Array}, x, Δx) = (@. y = x - Δx)
-@inline function __sub!!(y::AbstractArray, x, Δx)
-    ArrayInterface.can_setindex(y) && return (@. y = x - Δx)
-    return x .- Δx
-end
-
-@inline __add!!(x::Number, Δx) = x + Δx
-@inline __add!!(x::SArray, Δx) = x .+ Δx
-@inline __add!!(x::Union{MArray, Array}, Δx) = (x .+= Δx)
-@inline function __add!!(x::AbstractArray, Δx)
-    ArrayInterface.can_setindex(x) && return (x .+= Δx)
-    return x .+ Δx
-end
-
-@inline __add!!(::Number, x, Δx) = x + Δx
-@inline __add!!(::SArray, x, Δx) = x .+ Δx
-@inline __add!!(y::Union{MArray, Array}, x, Δx) = (@. y = x + Δx)
-@inline function __add!!(y::AbstractArray, x, Δx)
-    ArrayInterface.can_setindex(y) && return (@. y = x + Δx)
-    return x .+ Δx
-end
-
-@inline __copy(x::Union{Number, SArray}) = x
-@inline __copy(x::Union{Number, SArray}, _) = x
-@inline __copy(x::Union{MArray, Array}) = copy(x)
-@inline __copy(::Union{MArray, Array}, y) = copy(y)
-@inline function __copy(x::AbstractArray)
-    ArrayInterface.can_setindex(x) && return copy(x)
-    return x
-end
-@inline function __copy(x::AbstractArray, y)
-    ArrayInterface.can_setindex(x) && return copy(y)
-    return x
-end
-
-@inline __mul!!(::Union{Number, SArray}, A, b) = A * b
-@inline __mul!!(y::Union{MArray, Array}, A, b) = (mul!(y, A, b); y)
-@inline function __mul!!(y::AbstractArray, A, b)
-    ArrayInterface.can_setindex(y) && return (mul!(y, A, b); y)
-    return A * b
-end
-
-@inline __neg!!(x::Union{Number, SArray}) = -x
-@inline __neg!!(x::Union{MArray, Array}) = (@. x .*= -one(eltype(x)))
-@inline function __neg!!(x::AbstractArray)
-    ArrayInterface.can_setindex(x) && return (@. x .*= -one(eltype(x)))
-    return -x
-end
-
-@inline __ldiv!!(A, b::Union{Number, SArray}) = A \ b
-@inline __ldiv!!(A, b::Union{MArray, Array}) = (ldiv!(A, b); b)
-@inline function __ldiv!!(A, b::AbstractArray)
-    ArrayInterface.can_setindex(b) && return (ldiv!(A, b); b)
-    return A \ b
-end
-
-@inline __broadcast!!(y::Union{Number, SArray}, f::F, x, args...) where {F} = f.(x, args...)
-@inline function __broadcast!!(y::Union{MArray, Array}, f::F, x, args...) where {F}
-    @. y = f(x, args...)
-    return y
-end
-@inline function __broadcast!!(y::AbstractArray, f::F, x, args...) where {F}
-    ArrayInterface.can_setindex(y) && return (@. y = f(x, args...))
-    return f.(x, args...)
-end
-
-@inline __eval_f(prob, f, fx, x) = isinplace(prob) ? (f(fx, x); fx) : f(x)
+@inline __eval_f(prob, fx, x) = isinplace(prob) ? (prob.f(fx, x, prob.p); fx) :
+                                prob.f(x, prob.p)

From f22923177f95af945cc9a2f9a8ca6855d242d90f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 23 Nov 2023 02:50:28 -0500
Subject: [PATCH 224/375] Fix brent

---
 .../src/SimpleNonlinearSolve.jl               |   8 +-
 lib/SimpleNonlinearSolve/src/brent.jl         | 148 ++++++++----------
 2 files changed, 71 insertions(+), 85 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 208e0e15f..34d2b0728 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -40,7 +40,7 @@ include("klement.jl")
 include("bisection.jl")
 include("falsi.jl")
 include("ridder.jl")
-# include("brent.jl")
+include("brent.jl")
 # include("alefeld.jl")
 # include("itp.jl")
 
@@ -90,8 +90,8 @@ include("ridder.jl")
 # end
 
 export SimpleBroyden, SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson
-# SimpleDFSane, SimpleTrustRegion, SimpleHalley
-export Bisection, Falsi, Ridder
-# export , Brent, LBroyden, Alefeld, ITP
+# SimpleDFSane, SimpleTrustRegion, SimpleHalley, LBroyden
+export Bisection, Brent, Falsi, Ridder
+# export  Alefeld, ITP
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/brent.jl
index 1319ed979..75497f379 100644
--- a/lib/SimpleNonlinearSolve/src/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/brent.jl
@@ -1,127 +1,113 @@
 """
-`Brent()`
+    Brent()
 
-A non-allocating Brent method
+left non-allocating Brent method.
 """
 struct Brent <: AbstractBracketingAlgorithm end
 
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
-        maxiters = 1000, abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-        kwargs...)
+        maxiters = 1000, abstol = nothing, kwargs...)
+    @assert !isinplace(prob) "`Brent` only supports OOP problems."
     f = Base.Fix2(prob.f, prob.p)
-    a, b = prob.tspan
-    fa, fb = f(a), f(b)
-    ϵ = eps(convert(typeof(fa), 1.0))
+    left, right = prob.tspan
+    fl, fr = f(left), f(right)
+    ϵ = eps(convert(typeof(fl), 1))
 
-    if iszero(fa)
-        return SciMLBase.build_solution(prob, alg, a, fa;
-            retcode = ReturnCode.ExactSolutionLeft, left = a,
-            right = b)
-    elseif iszero(fb)
-        return SciMLBase.build_solution(prob, alg, b, fb;
-            retcode = ReturnCode.ExactSolutionRight, left = a,
-            right = b)
+    abstol = _get_tolerance(abstol,
+        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+
+    if iszero(fl)
+        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
+            left, right)
     end
-    if abs(fa) < abs(fb)
-        c = b
-        b = a
-        a = c
-        tmp = fa
-        fa = fb
-        fb = tmp
+
+    if iszero(fr)
+        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+            left, right)
     end
 
-    c = a
+    if abs(fl) < abs(fr)
+        c = right
+        right = left
+        left = c
+        tmp = fl
+        fl = fr
+        fr = tmp
+    end
+
+    c = left
     d = c
     i = 1
     cond = true
-    if !iszero(fb)
+    if !iszero(fr)
         while i < maxiters
             fc = f(c)
-            if fa != fc && fb != fc
+            if fl != fc && fr != fc
                 # Inverse quadratic interpolation
-                s = a * fb * fc / ((fa - fb) * (fa - fc)) +
-                    b * fa * fc / ((fb - fa) * (fb - fc)) +
-                    c * fa * fb / ((fc - fa) * (fc - fb))
+                s = left * fr * fc / ((fl - fr) * (fl - fc)) +
+                    right * fl * fc / ((fr - fl) * (fr - fc)) +
+                    c * fl * fr / ((fc - fl) * (fc - fr))
             else
                 # Secant method
-                s = b - fb * (b - a) / (fb - fa)
+                s = right - fr * (right - left) / (fr - fl)
             end
-            if (s < min((3 * a + b) / 4, b) || s > max((3 * a + b) / 4, b)) ||
-               (cond && abs(s - b) ≥ abs(b - c) / 2) ||
-               (!cond && abs(s - b) ≥ abs(c - d) / 2) ||
-               (cond && abs(b - c) ≤ ϵ) ||
+            if (s < min((3 * left + right) / 4, right) ||
+                s > max((3 * left + right) / 4, right)) ||
+               (cond && abs(s - right) ≥ abs(right - c) / 2) ||
+               (!cond && abs(s - right) ≥ abs(c - d) / 2) ||
+               (cond && abs(right - c) ≤ ϵ) ||
                (!cond && abs(c - d) ≤ ϵ)
                 # Bisection method
-                s = (a + b) / 2
-                (s == a || s == b) &&
-                    return SciMLBase.build_solution(prob, alg, a, fa;
+                s = (left + right) / 2
+                (s == left || s == right) &&
+                    return SciMLBase.build_solution(prob, alg, left, fl;
                         retcode = ReturnCode.FloatingPointLimit,
-                        left = a, right = b)
+                        left = left, right = right)
                 cond = true
             else
                 cond = false
             end
             fs = f(s)
-            if abs((b - a) / 2) < abstol
+            if abs((right - left) / 2) < abstol
                 return SciMLBase.build_solution(prob, alg, s, fs;
                     retcode = ReturnCode.Success,
-                    left = a, right = b)
+                    left = left, right = right)
             end
             if iszero(fs)
-                if b < a
-                    a = b
-                    fa = fb
+                if right < left
+                    left = right
+                    fl = fr
                 end
-                b = s
-                fb = fs
+                right = s
+                fr = fs
                 break
             end
-            if fa * fs < 0
+            if fl * fs < 0
                 d = c
-                c = b
-                b = s
-                fb = fs
+                c = right
+                right = s
+                fr = fs
             else
-                a = s
-                fa = fs
+                left = s
+                fl = fs
             end
-            if abs(fa) < abs(fb)
+            if abs(fl) < abs(fr)
                 d = c
-                c = b
-                b = a
-                a = c
-                fc = fb
-                fb = fa
-                fa = fc
+                c = right
+                right = left
+                left = c
+                fc = fr
+                fr = fl
+                fl = fc
             end
             i += 1
         end
     end
 
-    while i < maxiters
-        c = (a + b) / 2
-        if (c == a || c == b)
-            return SciMLBase.build_solution(prob, alg, a, fa;
-                retcode = ReturnCode.FloatingPointLimit,
-                left = a, right = b)
-        end
-        fc = f(c)
-        if abs((b - a) / 2) < abstol
-            return SciMLBase.build_solution(prob, alg, c, fc;
-                retcode = ReturnCode.Success,
-                left = a, right = b)
-        end
-        if iszero(fc)
-            b = c
-            fb = fc
-        else
-            a = c
-            fa = fc
-        end
-        i += 1
-    end
+    sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
+        maxiters = maxiters - i, prob, alg)
+
+    sol !== nothing && return sol
 
-    return SciMLBase.build_solution(prob, alg, a, fa; retcode = ReturnCode.MaxIters,
-        left = a, right = b)
+    return build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
 end

From 190ca22c9592793feff2d15639965bc353a15f17 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 23 Nov 2023 03:10:29 -0500
Subject: [PATCH 225/375] Fix the interval methods

---
 .../src/SimpleNonlinearSolve.jl               |  31 +++---
 lib/SimpleNonlinearSolve/src/alefeld.jl       |  68 ++++--------
 lib/SimpleNonlinearSolve/src/itp.jl           | 100 ++++++++----------
 lib/SimpleNonlinearSolve/src/ridder.jl        |   4 +-
 4 files changed, 88 insertions(+), 115 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 34d2b0728..695094805 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -41,8 +41,8 @@ include("bisection.jl")
 include("falsi.jl")
 include("ridder.jl")
 include("brent.jl")
-# include("alefeld.jl")
-# include("itp.jl")
+include("alefeld.jl")
+include("itp.jl")
 
 # AD
 # include("ad.jl")
@@ -62,9 +62,9 @@ include("brent.jl")
 
 # import PrecompileTools
 
-# PrecompileTools.@compile_workload begin
-#     for T in (Float32, Float64)
-#         prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
+@setup_workload begin
+    for T in (Float32, Float64)
+        # prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
 #         for alg in (SimpleNewtonRaphson, SimpleHalley, Broyden, Klement, SimpleTrustRegion,
 #             SimpleDFSane)
 #             solve(prob_no_brack, alg(), abstol = T(1e-2))
@@ -80,18 +80,19 @@ include("brent.jl")
 #         end
 #         =#
 
-#         prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
-#             T.((0.0, 2.0)),
-#             T(2))
-#         for alg in (Bisection, Falsi, Ridder, Brent, Alefeld, ITP)
-#             solve(prob_brack, alg(), abstol = T(1e-2))
-#         end
-#     end
-# end
+        prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
+            T.((0.0, 2.0)), T(2))
+        algs = [Bisection(), Falsi(), Ridder(), Brent(), Alefeld(), ITP()]
+        @compile_workload begin
+            for alg in algs
+                solve(prob_brack, alg, abstol = T(1e-2))
+            end
+        end
+    end
+end
 
 export SimpleBroyden, SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson
 # SimpleDFSane, SimpleTrustRegion, SimpleHalley, LBroyden
-export Bisection, Brent, Falsi, Ridder
-# export  Alefeld, ITP
+export Alefeld, Bisection, Brent, Falsi, ITP, Ridder
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/alefeld.jl
index 3d3b2ada8..39c984fd5 100644
--- a/lib/SimpleNonlinearSolve/src/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/alefeld.jl
@@ -1,5 +1,5 @@
 """
-`Alefeld()`
+    Alefeld()
 
 An implementation of algorithm 4.2 from [Alefeld](https://dl.acm.org/doi/10.1145/210089.210111).
 
@@ -8,24 +8,18 @@ algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal
 """
 struct Alefeld <: AbstractBracketingAlgorithm end
 
-function SciMLBase.solve(prob::IntervalNonlinearProblem,
-        alg::Alefeld, args...; abstol = nothing,
-        reltol = nothing,
-        maxiters = 1000, kwargs...)
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
+        maxiters = 1000, abstol = nothing, kwargs...)
     f = Base.Fix2(prob.f, prob.p)
     a, b = prob.tspan
     c = a - (b - a) / (f(b) - f(a)) * f(a)
 
     fc = f(c)
     (a == c || b == c) &&
-        return SciMLBase.build_solution(prob, alg, c, fc;
-            retcode = ReturnCode.FloatingPointLimit,
-            left = a,
-            right = b)
+        return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+            left = a, right = b)
     iszero(fc) &&
-        return SciMLBase.build_solution(prob, alg, c, fc;
-            retcode = ReturnCode.Success,
-            left = a,
+        return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success, left = a,
             right = b)
     a, b, d = _bracket(f, a, b, c)
     e = zero(a)   # Set e as 0 before iteration to avoid a non-value f(e)
@@ -44,15 +38,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
         end
         ē, fc = d, f(c)
         (a == c || b == c) &&
-            return SciMLBase.build_solution(prob, alg, c, fc;
-                retcode = ReturnCode.FloatingPointLimit,
-                left = a,
-                right = b)
+            return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+                left = a, right = b)
         iszero(fc) &&
-            return SciMLBase.build_solution(prob, alg, c, fc;
-                retcode = ReturnCode.Success,
-                left = a,
-                right = b)
+            return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
+                left = a, right = b)
         ā, b̄, d̄ = _bracket(f, a, b, c)
 
         # The second bracketing block
@@ -67,15 +57,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
         end
         fc = f(c)
         (ā == c || b̄ == c) &&
-            return SciMLBase.build_solution(prob, alg, c, fc;
-                retcode = ReturnCode.FloatingPointLimit,
-                left = ā,
-                right = b̄)
+            return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+                left = ā, right = b̄)
         iszero(fc) &&
-            return SciMLBase.build_solution(prob, alg, c, fc;
-                retcode = ReturnCode.Success,
-                left = ā,
-                right = b̄)
+            return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
+                left = ā, right = b̄)
         ā, b̄, d̄ = _bracket(f, ā, b̄, c)
 
         # The third bracketing block
@@ -90,15 +76,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
         end
         fc = f(c)
         (ā == c || b̄ == c) &&
-            return SciMLBase.build_solution(prob, alg, c, fc;
-                retcode = ReturnCode.FloatingPointLimit,
-                left = ā,
-                right = b̄)
+            return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+                left = ā, right = b̄)
         iszero(fc) &&
-            return SciMLBase.build_solution(prob, alg, c, fc;
-                retcode = ReturnCode.Success,
-                left = ā,
-                right = b̄)
+            return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
+                left = ā, right = b̄)
         ā, b̄, d = _bracket(f, ā, b̄, c)
 
         # The last bracketing block
@@ -109,15 +91,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
             c = 0.5 * (ā + b̄)
             fc = f(c)
             (ā == c || b̄ == c) &&
-                return SciMLBase.build_solution(prob, alg, c, fc;
-                    retcode = ReturnCode.FloatingPointLimit,
-                    left = ā,
-                    right = b̄)
+                return build_solution(prob, alg, c, fc;
+                    retcode = ReturnCode.FloatingPointLimit, left = ā, right = b̄)
             iszero(fc) &&
-                return SciMLBase.build_solution(prob, alg, c, fc;
-                    retcode = ReturnCode.Success,
-                    left = ā,
-                    right = b̄)
+                return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
+                    left = ā, right = b̄)
             a, b, d = _bracket(f, ā, b̄, c)
         end
     end
@@ -131,7 +109,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem,
     fc = f(c)
 
     # Reuturn solution when run out of max interation
-    return SciMLBase.build_solution(prob, alg, c, fc; retcode = ReturnCode.MaxIters,
+    return build_solution(prob, alg, c, fc; retcode = ReturnCode.MaxIters,
         left = a, right = b)
 end
 
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/itp.jl
index 933995cec..fd46de6c3 100644
--- a/lib/SimpleNonlinearSolve/src/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/itp.jl
@@ -1,33 +1,29 @@
 """
-```julia
-ITP(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
-```
+    ITP(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
 
 ITP (Interpolate Truncate & Project)
 
-Use the [ITP method](https://en.wikipedia.org/wiki/ITP_method) to find
-a root of a bracketed function, with a convergence rate between 1 and 1.62.
+Use the [ITP method](https://en.wikipedia.org/wiki/ITP_method) to find a root of a bracketed
+function, with a convergence rate between 1 and 1.62.
 
-This method was introduced in the paper "An Enhancement of the Bisection Method
-Average Performance Preserving Minmax Optimality"
-(https://doi.org/10.1145/3423597) by I. F. D. Oliveira and R. H. C. Takahashi.
+This method was introduced in the paper "An Enhancement of the Bisection Method Average
+Performance Preserving Minmax Optimality" (https://doi.org/10.1145/3423597) by
+I. F. D. Oliveira and R. H. C. Takahashi.
 
 # Tuning Parameters
 
 The following keyword parameters are accepted.
 
-  - `n₀::Int = 1`, the 'slack'. Must not be negative.\n
-    When n₀ = 0 the worst-case is identical to that of bisection,
-    but increacing n₀ provides greater oppotunity for superlinearity.
-  - `κ₁::Float64 = 0.1`. Must not be negative.\n
-    The recomended value is `0.2/(x₂ - x₁)`.
-    Lower values produce tighter asymptotic behaviour, while higher values
-    improve the steady-state behaviour when truncation is not helpful.
-  - `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62).\n
-    Higher values allow for a greater convergence rate,
-    but also make the method more succeptable to worst-case performance.
-    In practice, κ=1,2 seems to work well due to the computational simplicity,
-    as κ₂ is used as an exponent in the method.
+  - `n₀::Int = 1`, the 'slack'. Must not be negative. When n₀ = 0 the worst-case is
+    identical to that of bisection, but increacing n₀ provides greater oppotunity for
+    superlinearity.
+  - `κ₁::Float64 = 0.1`. Must not be negative. The recomended value is `0.2/(x₂ - x₁)`.
+    Lower values produce tighter asymptotic behaviour, while higher values improve the
+    steady-state behaviour when truncation is not helpful.
+  - `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62). Higher values allow for a greater
+    convergence rate, but also make the method more succeptable to worst-case performance.
+    In practice, κ=1,2 seems to work well due to the computational simplicity, as κ₂ is used
+    as an exponent in the method.
 
 ### Worst Case Performance
 
@@ -36,44 +32,45 @@ n½ + `n₀` iterations, where n½ is the number of iterations using bisection
 
 ### Asymptotic Performance
 
-If `f` is twice differentiable and the root is simple,
-then with `n₀` > 0 the convergence rate is √`κ₂`.
+If `f` is twice differentiable and the root is simple, then with `n₀` > 0 the convergence
+rate is √`κ₂`.
 """
 struct ITP{T} <: AbstractBracketingAlgorithm
     k1::T
     k2::T
     n0::Int
     function ITP(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
-        if k1 < 0
-            error("Hyper-parameter κ₁ should not be negative")
-        end
-        if n0 < 0
-            error("Hyper-parameter n₀ should not be negative")
-        end
+        k1 < 0 && error("Hyper-parameter κ₁ should not be negative")
+        n0 < 0 && error("Hyper-parameter n₀ should not be negative")
         if k2 < 1 || k2 > (1.5 + sqrt(5) / 2)
-            ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where ϕ ≈ 1.618... is the golden ratio")
+            throw(ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where \
+                                 ϕ ≈ 1.618... is the golden ratio"))
         end
         T = promote_type(eltype(k1), eltype(k2))
         return new{T}(k1, k2, n0)
     end
 end
 
-function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
-        args...; abstol = min(eps(prob.tspan[1]), eps(prob.tspan[2])),
-        maxiters = 1000, kwargs...)
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
+        maxiters = 1000, abstol = nothing, kwargs...)
+    @assert !isinplace(prob) "`Bisection` only supports OOP problems."
     f = Base.Fix2(prob.f, prob.p)
-    left, right = prob.tspan # a and b
+    left, right = prob.tspan
     fl, fr = f(left), f(right)
-    ϵ = abstol
+
+    abstol = _get_tolerance(abstol,
+        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+
     if iszero(fl)
-        return SciMLBase.build_solution(prob, alg, left, fl;
-            retcode = ReturnCode.ExactSolutionLeft, left = left,
-            right = right)
-    elseif iszero(fr)
-        return SciMLBase.build_solution(prob, alg, right, fr;
-            retcode = ReturnCode.ExactSolutionRight, left = left,
-            right = right)
+        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
+            left, right)
     end
+
+    if iszero(fr)
+        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+            left, right)
+    end
+    ϵ = abstol
     #defining variables/cache
     k1 = alg.k1
     k2 = alg.k2
@@ -112,9 +109,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
         end
 
         if abs((left - right) / 2) < ϵ
-            return SciMLBase.build_solution(prob, alg, mid, f(mid);
-                retcode = ReturnCode.Success,
-                left = left, right = right)
+            return build_solution(prob, alg, mid, f(mid); retcode = ReturnCode.Success,
+                left, right)
         end
 
         ## Update ##
@@ -130,20 +126,18 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP,
             left = xp
             fl = yp
         else
-            return SciMLBase.build_solution(prob, alg, xp, yps;
-                retcode = ReturnCode.Success, left = xp,
-                right = xp)
+            return build_solution(prob, alg, xp, yps; retcode = ReturnCode.Success,
+                left = xp, right = xp)
         end
         i += 1
         mid = (left + right) / 2
         ϵ_s /= 2
 
-        if nextfloat_tdir(left, prob.tspan...) == right
-            return SciMLBase.build_solution(prob, alg, left, fl;
-                retcode = ReturnCode.FloatingPointLimit, left = left,
-                right = right)
+        if __nextfloat_tdir(left, prob.tspan...) == right
+            return build_solution(prob, alg, left, fl; left, right,
+                retcode = ReturnCode.FloatingPointLimit)
         end
     end
-    return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
-        left = left, right = right)
+
+    return build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
 end
diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/ridder.jl
index 0bed8ee75..11b7604e1 100644
--- a/lib/SimpleNonlinearSolve/src/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/ridder.jl
@@ -30,7 +30,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
     if !iszero(fr)
         while i < maxiters
             mid = (left + right) / 2
-            if (mid == left || mid == right)
+            (mid == left || mid == right) &&
                 return build_solution(prob, alg, left, fl; left, right,
                     retcode = ReturnCode.FloatingPointLimit)
             fm = f(mid)
@@ -70,7 +70,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
 
     sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
         maxiters = maxiters - i, prob, alg)
-    sol !== nothing && return sol
+        sol !== nothing && return sol
 
     return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
         left, right)

From b50ece4301342fe9145301d00974d299f38cdc9d Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 23 Nov 2023 19:18:29 -0500
Subject: [PATCH 226/375] Move things around a bit

---
 .../src/SimpleNonlinearSolve.jl               | 72 +++++++++----------
 .../src/{ => bracketing}/alefeld.jl           |  0
 .../src/{ => bracketing}/bisection.jl         |  0
 .../src/{ => bracketing}/brent.jl             |  0
 .../src/{ => bracketing}/falsi.jl             |  0
 .../src/{ => bracketing}/itp.jl               |  0
 .../src/{ => bracketing}/ridder.jl            |  0
 .../src/{ => nlsolve}/broyden.jl              |  0
 .../src/{ => nlsolve}/dfsane.jl               |  0
 .../src/{ => nlsolve}/halley.jl               |  0
 .../src/{ => nlsolve}/klement.jl              |  0
 .../src/{ => nlsolve}/lbroyden.jl             |  0
 .../src/{ => nlsolve}/raphson.jl              |  0
 .../src/{ => nlsolve}/trustRegion.jl          |  0
 14 files changed, 36 insertions(+), 36 deletions(-)
 rename lib/SimpleNonlinearSolve/src/{ => bracketing}/alefeld.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => bracketing}/bisection.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => bracketing}/brent.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => bracketing}/falsi.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => bracketing}/itp.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => bracketing}/ridder.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => nlsolve}/broyden.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => nlsolve}/dfsane.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => nlsolve}/halley.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => nlsolve}/klement.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => nlsolve}/lbroyden.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => nlsolve}/raphson.jl (100%)
 rename lib/SimpleNonlinearSolve/src/{ => nlsolve}/trustRegion.jl (100%)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 695094805..ab7026bf8 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -28,57 +28,57 @@ include("utils.jl")
 include("rewrite_inplace.jl")
 
 # Nonlinear Solvera
-include("raphson.jl")
-include("broyden.jl")
-# include("lbroyden.jl")
-include("klement.jl")
-# include("trustRegion.jl")
-# include("halley.jl")
-# include("dfsane.jl")
+include("nlsolve/raphson.jl")
+include("nlsolve/broyden.jl")
+# include("nlsolve/lbroyden.jl")
+include("nlsolve/klement.jl")
+# include("nlsolve/trustRegion.jl")
+# include("nlsolve/halley.jl")
+# include("nlsolve/dfsane.jl")
 
 # Interval Nonlinear Solvers
-include("bisection.jl")
-include("falsi.jl")
-include("ridder.jl")
-include("brent.jl")
-include("alefeld.jl")
-include("itp.jl")
+include("bracketing/bisection.jl")
+include("bracketing/falsi.jl")
+include("bracketing/ridder.jl")
+include("bracketing/brent.jl")
+include("bracketing/alefeld.jl")
+include("bracketing/itp.jl")
 
 # AD
 # include("ad.jl")
 
-# ## Default algorithm
+## Default algorithm
 
-# # Set the default bracketing method to ITP
+# Set the default bracketing method to ITP
 
-# function SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...)
-#     SciMLBase.solve(prob, ITP(); kwargs...)
-# end
+function SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...)
+    return solve(prob, ITP(); kwargs...)
+end
 
-# function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing,
-#     args...; kwargs...)
-#     SciMLBase.solve(prob, ITP(), args...; kwargs...)
-# end
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing,
+        args...; kwargs...)
+    return solve(prob, ITP(), args...; kwargs...)
+end
 
 # import PrecompileTools
 
 @setup_workload begin
     for T in (Float32, Float64)
         # prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-#         for alg in (SimpleNewtonRaphson, SimpleHalley, Broyden, Klement, SimpleTrustRegion,
-#             SimpleDFSane)
-#             solve(prob_no_brack, alg(), abstol = T(1e-2))
-#         end
-
-#         #=
-#         for alg in (SimpleNewtonRaphson,)
-#             for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
-#                 u0 = T.(.1)
-#                 probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
-#                 solve(probN, alg(), tol = T(1e-2))
-#             end
-#         end
-#         =#
+        #         for alg in (SimpleNewtonRaphson, SimpleHalley, Broyden, Klement, SimpleTrustRegion,
+        #             SimpleDFSane)
+        #             solve(prob_no_brack, alg(), abstol = T(1e-2))
+        #         end
+
+        #         #=
+        #         for alg in (SimpleNewtonRaphson,)
+        #             for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
+        #                 u0 = T.(.1)
+        #                 probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
+        #                 solve(probN, alg(), tol = T(1e-2))
+        #             end
+        #         end
+        #         =#
 
         prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
             T.((0.0, 2.0)), T(2))
diff --git a/lib/SimpleNonlinearSolve/src/alefeld.jl b/lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/alefeld.jl
rename to lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl
diff --git a/lib/SimpleNonlinearSolve/src/bisection.jl b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/bisection.jl
rename to lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
diff --git a/lib/SimpleNonlinearSolve/src/brent.jl b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/brent.jl
rename to lib/SimpleNonlinearSolve/src/bracketing/brent.jl
diff --git a/lib/SimpleNonlinearSolve/src/falsi.jl b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/falsi.jl
rename to lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
diff --git a/lib/SimpleNonlinearSolve/src/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/itp.jl
rename to lib/SimpleNonlinearSolve/src/bracketing/itp.jl
diff --git a/lib/SimpleNonlinearSolve/src/ridder.jl b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/ridder.jl
rename to lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/broyden.jl
rename to lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/dfsane.jl
rename to lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/halley.jl
rename to lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/klement.jl
rename to lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/lbroyden.jl
rename to lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/raphson.jl
rename to lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
diff --git a/lib/SimpleNonlinearSolve/src/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
similarity index 100%
rename from lib/SimpleNonlinearSolve/src/trustRegion.jl
rename to lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl

From 62d99e03a597ae2cac822337b829ab172b43a8f1 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 23 Nov 2023 22:51:03 -0500
Subject: [PATCH 227/375] Move out the @bb macro into a separate package

---
 lib/SimpleNonlinearSolve/Project.toml         |   1 +
 .../src/SimpleNonlinearSolve.jl               |  10 +-
 .../src/bracketing/ridder.jl                  |   2 +-
 .../src/nlsolve/klement.jl                    |   8 +-
 .../src/rewrite_inplace.jl                    | 161 ------------------
 5 files changed, 10 insertions(+), 172 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/src/rewrite_inplace.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 75af93414..8e6b0f510 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -11,6 +11,7 @@ DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index ab7026bf8..cdfc95b5c 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -4,28 +4,25 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
 
 @recompile_invalidations begin
     using ADTypes,
-        ArrayInterface, ConcreteStructs, DiffEqBase, Reexport, LinearAlgebra,
-        SciMLBase
+        ArrayInterface, ConcreteStructs, DiffEqBase, Reexport, LinearAlgebra, SciMLBase
 
     import DiffEqBase: AbstractNonlinearTerminationMode,
         AbstractSafeNonlinearTerminationMode, AbstractSafeBestNonlinearTerminationMode,
         NonlinearSafeTerminationReturnCode, get_termination_mode
     using FiniteDiff, ForwardDiff
     import ForwardDiff: Dual
+    import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
     import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace
     import StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray
 end
 
 @reexport using ADTypes, SciMLBase
 
-# const NNlibExtLoaded = Ref{Bool}(false)
-
 abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorithm end
 abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 abstract type AbstractNewtonAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 
 include("utils.jl")
-include("rewrite_inplace.jl")
 
 # Nonlinear Solvera
 include("nlsolve/raphson.jl")
@@ -50,7 +47,6 @@ include("bracketing/itp.jl")
 ## Default algorithm
 
 # Set the default bracketing method to ITP
-
 function SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...)
     return solve(prob, ITP(); kwargs...)
 end
@@ -60,8 +56,6 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing,
     return solve(prob, ITP(), args...; kwargs...)
 end
 
-# import PrecompileTools
-
 @setup_workload begin
     for T in (Float32, Float64)
         # prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
index 11b7604e1..20e0db489 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
@@ -70,7 +70,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
 
     sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
         maxiters = maxiters - i, prob, alg)
-        sol !== nothing && return sol
+    sol !== nothing && return sol
 
     return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
         left, right)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index 7b9a878ad..56d6ccd55 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -54,7 +54,11 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
         end
 
         @bb copyto!(δx, fprev)
-        δx = __ldiv!!(F_, δx)
+        if setindex_trait(δx) === CanSetindex()
+            ldiv!(F_, δx)
+        else
+            δx = F_ \ δx
+        end
         @bb @. x = xo - δx
         fx = __eval_f(prob, fx, x)
 
@@ -74,7 +78,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
         @bb δx² = J × vec(δx)
         @bb @. δf = (δf - δx²) / d
 
-        _vδf, _vδx = vec(δf), vec(δx)
+        _vδf, _vδx = _vec(δf), _vec(δx)
         @bb J_cache = _vδf × transpose(_vδx)
         @bb @. J_cache *= J
         @bb J_cache2 = J_cache × J
diff --git a/lib/SimpleNonlinearSolve/src/rewrite_inplace.jl b/lib/SimpleNonlinearSolve/src/rewrite_inplace.jl
deleted file mode 100644
index f0d80af23..000000000
--- a/lib/SimpleNonlinearSolve/src/rewrite_inplace.jl
+++ /dev/null
@@ -1,161 +0,0 @@
-# Take a inplace code and rewrite it to be maybe-inplace
-# I will take this code out into a separate package because this is useful even in
-# NonlinearSolve.jl
-function __bangbang(M, expr; depth = 1)
-    new_expr = nothing
-    if expr.head == :call
-        @assert length(expr.args)≥2 "Expected a function call with atleast 1 argument. \
-                                     Got `$(expr)`."
-        f, a, args... = expr.args
-        g = get(OP_MAPPING, f, nothing)
-        if f == :copy && length(args) == 0
-            # Special case for copy with single argument
-            new_expr = :($(g)($(setindex_trait)($(a)), $(a)))
-        elseif g !== nothing
-            new_expr = :($(a) = $(g)($(setindex_trait)($(a)), $(a), $(args...)))
-        end
-    elseif expr.head == :(=)
-        a, rhs_expr = expr.args
-        if rhs_expr.head == :call
-            f, b, args... = rhs_expr.args
-            g = get(OP_MAPPING, f, nothing)
-            if g !== nothing
-                new_expr = :($(a) = $(g)($(setindex_trait)($(b)), $(b), $(args...)))
-            elseif f == :×
-                @debug "Custom operator `×` detected in `$(expr)`."
-                c, args... = args
-                @assert length(args)==0 "Expected `×` to have only 2 arguments. \
-                                        Got `$(expr)`."
-                is_b_vec = b isa Expr && b.head == :call && b.args[1] == :vec
-                is_c_vec = c isa Expr && c.head == :call && c.args[1] == :vec
-                a_sym = gensym("a")
-                if is_b_vec
-                    if is_c_vec
-                        error("2 `vec`s detected with `×` in `$(expr)`. Use `dot` instead.")
-                    else
-                        new_expr = quote
-                            if $(setindex_trait)($(a)) === CanSetindex()
-                                $(a_sym) = $(_vec)($a)
-                                mul!($(a_sym), $(_vec)($(b.args[2])), $(c))
-                                $(a) = $(_restructure)($a, $(a_sym))
-                            else
-                                $(a) = $(_restructure)($a, $(_vec)($(b.args[2])) * $(c))
-                            end
-                        end
-                    end
-                else
-                    if is_c_vec
-                        new_expr = quote
-                            if $(setindex_trait)($(a)) === CanSetindex()
-                                $(a_sym) = $(_vec)($a)
-                                mul!($(a), $(b), $(_vec)($(c.args[2])))
-                                $(a) = $(_restructure)($a, $(a_sym))
-                            else
-                                $(a) = $(_restructure)($a, $(b) * $(_vec)($(c.args[2])))
-                            end
-                        end
-                    else
-                        new_expr = quote
-                            if $(setindex_trait)($(a)) === CanSetindex()
-                                mul!($(a), $(b), $(c))
-                            else
-                                $(a) = $(b) * $(c)
-                            end
-                        end
-                    end
-                end
-            end
-        end
-    elseif expr.head == :(.=)
-        a, rhs_expr = expr.args
-        if rhs_expr isa Expr && rhs_expr.head == :(.)
-            f, arg_expr = rhs_expr.args
-            # f_broadcast = :(Base.Broadcast.BroadcastFunction($(f)))
-            new_expr = quote
-                if $(setindex_trait)($(a)) === CanSetindex()
-                    broadcast!($(f), $(a), $(arg_expr)...)
-                else
-                    $(a) = broadcast($(f), $(arg_expr)...)
-                end
-            end
-        end
-    elseif expr.head == :macrocall
-        # For @__dot__ there is a easier alternative
-        if expr.args[1] == Symbol("@__dot__")
-            main_expr = last(expr.args)
-            if main_expr isa Expr && main_expr.head == :(=)
-                a, rhs_expr = main_expr.args
-                new_expr = quote
-                    if $(setindex_trait)($(a)) === CanSetindex()
-                        @. $(main_expr)
-                    else
-                        $(a) = @. $(rhs_expr)
-                    end
-                end
-            end
-        end
-        if new_expr === nothing
-            new_expr = __bangbang(M, Base.macroexpand(M, expr; recursive = true);
-                depth = depth + 1)
-        end
-    else
-        f = expr.head # Things like :.-=, etc.
-        a, args... = expr.args
-        g = get(OP_MAPPING, f, nothing)
-        if g !== nothing
-            new_expr = :($(a) = $(g)($(setindex_trait)($(a)), $(a), $(args...)))
-        end
-    end
-    if new_expr !== nothing
-        if depth == 1
-            @debug "Replacing `$(expr)` with `$(new_expr)`"
-            return esc(new_expr)
-        else
-            return new_expr
-        end
-    end
-    error("`$(expr)` cannot be handled. Check the documentation for allowed expressions.")
-end
-
-macro bangbang(expr)
-    return __bangbang(__module__, expr)
-end
-
-# `bb` is the short form of bang-bang
-macro bb(expr)
-    return __bangbang(__module__, expr)
-end
-
-# Is Mutable or Not?
-abstract type AbstractMaybeSetindex end
-struct CannotSetindex <: AbstractMaybeSetindex end
-struct CanSetindex <: AbstractMaybeSetindex end
-
-# Common types should overload this via extensions, else it butchers type-inference
-setindex_trait(::Union{Number, SArray}) = CannotSetindex()
-setindex_trait(::Union{MArray, Array}) = CanSetindex()
-setindex_trait(A) = ifelse(ArrayInterface.can_setindex(A), CanSetindex(), CannotSetindex())
-
-# Operations
-const OP_MAPPING = Dict{Symbol, Symbol}(:copyto! => :__copyto!!,
-    :.-= => :__sub!!,
-    :.+= => :__add!!,
-    :.*= => :__mul!!,
-    :./= => :__div!!,
-    :copy => :__copy)
-
-@inline __copyto!!(::CannotSetindex, x, y) = y
-@inline __copyto!!(::CanSetindex, x, y) = (copyto!(x, y); x)
-
-@inline __broadcast!!(::CannotSetindex, op, x, args...) = broadcast(op, args...)
-@inline __broadcast!!(::CanSetindex, op, x, args...) = (broadcast!(op, x, args...); x)
-
-@inline __sub!!(S, x, args...) = __broadcast!!(S, -, x, x, args...)
-@inline __add!!(S, x, args...) = __broadcast!!(S, +, x, x, args...)
-@inline __mul!!(S, x, args...) = __broadcast!!(S, *, x, x, args...)
-@inline __div!!(S, x, args...) = __broadcast!!(S, /, x, x, args...)
-
-@inline __copy(::CannotSetindex, x) = x
-@inline __copy(::CanSetindex, x) = copy(x)
-@inline __copy(::CannotSetindex, x, y) = y
-@inline __copy(::CanSetindex, x, y) = copy(y)

From 12006d11aab4c0369a1dd0c8ef331a553e732932 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 23 Nov 2023 23:07:45 -0500
Subject: [PATCH 228/375] Reenable some more compilation

---
 .../src/SimpleNonlinearSolve.jl               | 42 ++++++++++++-------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index cdfc95b5c..b74090a9f 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -58,21 +58,33 @@ end
 
 @setup_workload begin
     for T in (Float32, Float64)
-        # prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-        #         for alg in (SimpleNewtonRaphson, SimpleHalley, Broyden, Klement, SimpleTrustRegion,
-        #             SimpleDFSane)
-        #             solve(prob_no_brack, alg(), abstol = T(1e-2))
-        #         end
-
-        #         #=
-        #         for alg in (SimpleNewtonRaphson,)
-        #             for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
-        #                 u0 = T.(.1)
-        #                 probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
-        #                 solve(probN, alg(), tol = T(1e-2))
-        #             end
-        #         end
-        #         =#
+        prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
+        algs = [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement()]
+
+        @compile_workload begin
+            for alg in algs
+                solve(prob_no_brack, alg, abstol = T(1e-2))
+            end
+        end
+
+        prob_no_brack = NonlinearProblem{true}((du, u, p) -> du .= u .* u .- p,
+            T.([1.0, 1.0]), T(2))
+
+        @compile_workload begin
+            for alg in algs
+                solve(prob_no_brack, alg, abstol = T(1e-2))
+            end
+        end
+
+        #=
+        for alg in (SimpleNewtonRaphson,)
+            for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
+                u0 = T.(.1)
+                probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
+                solve(probN, alg(), tol = T(1e-2))
+            end
+        end
+        =#
 
         prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
             T.((0.0, 2.0)), T(2))

From 1b1029e84c096c4aefbcd0620281173a0e705a8d Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 23 Nov 2023 23:10:26 -0500
Subject: [PATCH 229/375] bad rebase

---
 .../src/batched/raphson.jl                    | 92 -------------------
 1 file changed, 92 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/src/batched/raphson.jl

diff --git a/lib/SimpleNonlinearSolve/src/batched/raphson.jl b/lib/SimpleNonlinearSolve/src/batched/raphson.jl
deleted file mode 100644
index 7bc7b8c4a..000000000
--- a/lib/SimpleNonlinearSolve/src/batched/raphson.jl
+++ /dev/null
@@ -1,92 +0,0 @@
-struct BatchedSimpleNewtonRaphson{CS, AD, FDT, TC <: NLSolveTerminationCondition} <:
-       AbstractBatchedNonlinearSolveAlgorithm
-    termination_condition::TC
-end
-
-alg_autodiff(alg::BatchedSimpleNewtonRaphson{CS, AD, FDT}) where {CS, AD, FDT} = AD
-diff_type(alg::BatchedSimpleNewtonRaphson{CS, AD, FDT}) where {CS, AD, FDT} = FDT
-
-function BatchedSimpleNewtonRaphson(; chunk_size = Val{0}(),
-        autodiff = Val{true}(),
-        diff_type = Val{:forward},
-        termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-            abstol = nothing,
-            reltol = nothing))
-    return BatchedSimpleNewtonRaphson{SciMLBase._unwrap_val(chunk_size),
-        SciMLBase._unwrap_val(autodiff),
-        SciMLBase._unwrap_val(diff_type), typeof(termination_condition)}(termination_condition)
-end
-
-function SciMLBase.__solve(prob::NonlinearProblem, alg::BatchedSimpleNewtonRaphson;
-        abstol = nothing, reltol = nothing, maxiters = 1000, kwargs...)
-    iip = SciMLBase.isinplace(prob)
-    iip &&
-        @assert alg_autodiff(alg) "Inplace BatchedSimpleNewtonRaphson currently only supports autodiff."
-    u, f, reconstruct = _construct_batched_problem_structure(prob)
-
-    tc = alg.termination_condition
-    mode = DiffEqBase.get_termination_mode(tc)
-
-    storage = _get_storage(mode, u)
-
-    xₙ, xₙ₋₁ = copy(u), copy(u)
-    T = eltype(u)
-
-    atol = _get_tolerance(abstol, tc.abstol, T)
-    rtol = _get_tolerance(reltol, tc.reltol, T)
-    termination_condition = tc(storage)
-
-    if iip
-        𝓙 = similar(xₙ, length(xₙ), length(xₙ))
-        fₙ = similar(xₙ)
-        jac_cfg = ForwardDiff.JacobianConfig(f, fₙ, xₙ)
-    end
-
-    for i in 1:maxiters
-        if iip
-            value_derivative!(𝓙, fₙ, f, xₙ, jac_cfg)
-        else
-            if alg_autodiff(alg)
-                fₙ, 𝓙 = value_derivative(f, xₙ)
-            else
-                fₙ = f(xₙ)
-                𝓙 = FiniteDiff.finite_difference_jacobian(f,
-                    xₙ,
-                    diff_type(alg),
-                    eltype(xₙ),
-                    fₙ)
-            end
-        end
-
-        iszero(fₙ) && return DiffEqBase.build_solution(prob,
-            alg,
-            reconstruct(xₙ),
-            reconstruct(fₙ);
-            retcode = ReturnCode.Success)
-
-        δx = reshape(𝓙 \ vec(fₙ), size(xₙ))
-        xₙ .-= δx
-
-        if termination_condition(fₙ, xₙ, xₙ₋₁, atol, rtol)
-            retcode, xₙ, fₙ = _result_from_storage(storage, xₙ, fₙ, f, mode, iip)
-            return DiffEqBase.build_solution(prob,
-                alg,
-                reconstruct(xₙ),
-                reconstruct(fₙ);
-                retcode)
-        end
-
-        xₙ₋₁ .= xₙ
-    end
-
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        xₙ = storage.u
-        @maybeinplace iip fₙ=f(xₙ)
-    end
-
-    return DiffEqBase.build_solution(prob,
-        alg,
-        reconstruct(xₙ),
-        reconstruct(fₙ);
-        retcode = ReturnCode.MaxIters)
-end

From 3c3e401eb8587e578674e19a5b3ccc04c9804aac Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 24 Nov 2023 01:25:56 -0500
Subject: [PATCH 230/375] More robust and allocated version of TrustRegion

---
 .../src/SimpleNonlinearSolve.jl               |  10 +-
 .../src/nlsolve/raphson.jl                    |  10 +-
 .../src/nlsolve/trustRegion.jl                | 221 ++++++++----------
 lib/SimpleNonlinearSolve/src/utils.jl         |  25 --
 4 files changed, 107 insertions(+), 159 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index b74090a9f..79a4d3f53 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -29,7 +29,7 @@ include("nlsolve/raphson.jl")
 include("nlsolve/broyden.jl")
 # include("nlsolve/lbroyden.jl")
 include("nlsolve/klement.jl")
-# include("nlsolve/trustRegion.jl")
+include("nlsolve/trustRegion.jl")
 # include("nlsolve/halley.jl")
 # include("nlsolve/dfsane.jl")
 
@@ -59,7 +59,8 @@ end
 @setup_workload begin
     for T in (Float32, Float64)
         prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-        algs = [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement()]
+        algs = [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement(),
+            SimpleTrustRegion()]
 
         @compile_workload begin
             for alg in algs
@@ -97,8 +98,9 @@ end
     end
 end
 
-export SimpleBroyden, SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson
-# SimpleDFSane, SimpleTrustRegion, SimpleHalley, LBroyden
+export SimpleBroyden,
+    SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson, SimpleTrustRegion
+# SimpleDFSane, SimpleHalley, LBroyden
 export Alefeld, Bisection, Brent, Falsi, ITP, Ridder
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
index 1b63656de..3d8debfe4 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
@@ -16,12 +16,10 @@ and static array problems.
   - `autodiff`: determines the backend used for the Jacobian. Defaults to
     `AutoForwardDiff()`. Valid choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
 """
-@concrete struct SimpleNewtonRaphson <: AbstractNewtonAlgorithm
-    ad
+@kwdef @concrete struct SimpleNewtonRaphson <: AbstractNewtonAlgorithm
+    autodiff = AutoForwardDiff()
 end
 
-SimpleNewtonRaphson(; autodiff = AutoForwardDiff()) = SimpleNewtonRaphson(autodiff)
-
 const SimpleGaussNewton = SimpleNewtonRaphson
 
 function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
@@ -30,13 +28,13 @@ function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresPr
     @bb x = copy(float(prob.u0))
     fx = _get_fx(prob, x)
     @bb xo = copy(x)
-    J, jac_cache = jacobian_cache(alg.ad, prob.f, fx, x, prob.p)
+    J, jac_cache = jacobian_cache(alg.autodiff, prob.f, fx, x, prob.p)
 
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
 
     for i in 1:maxiters
-        fx, dfx = value_and_jacobian(alg.ad, prob.f, fx, x, prob.p, jac_cache; J)
+        fx, dfx = value_and_jacobian(alg.autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
         if i == 1
             if iszero(fx)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index d644f5fb7..2420b7226 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -1,27 +1,17 @@
 """
-    SimpleTrustRegion(; chunk_size = Val{0}(), autodiff = Val{true}(),
-                        diff_type = Val{:forward}, max_trust_radius::Real = 0.0,
+    SimpleTrustRegion(; autodiff = AutoForwardDiff(), max_trust_radius::Real = 0.0,
                         initial_trust_radius::Real = 0.0, step_threshold::Real = 0.1,
                         shrink_threshold::Real = 0.25, expand_threshold::Real = 0.75,
                         shrink_factor::Real = 0.25, expand_factor::Real = 2.0,
                         max_shrink_times::Int = 32)
 
-A low-overhead implementation of a trust-region solver.
+A low-overhead implementation of a trust-region solver. This method is non-allocating on
+scalar and static array problems.
 
 ### Keyword Arguments
 
-  - `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
-    system. This allows for multiple derivative columns to be computed simultaneously,
-    improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
-    default chunk size mechanism. For more details, see the documentation for
-    [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
-  - `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
-    Note that this argument is ignored if an analytical Jacobian is passed; as that will be
-    used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
-    If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
-  - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
-    `Val{:forward}` for forward finite differences. For more details on the choices, see the
-    [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+  - `autodiff`: determines the backend used for the Jacobian. Defaults to
+    `AutoForwardDiff()`. Valid choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
   - `max_trust_radius`: the maximum radius of the trust region. Defaults to
     `max(norm(f(u0)), maximum(u0) - minimum(u0))`.
   - `initial_trust_radius`: the initial trust region radius. Defaults to
@@ -47,143 +37,126 @@ A low-overhead implementation of a trust-region solver.
   - `max_shrink_times`: the maximum number of times to shrink the trust region radius in a
     row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`.
 """
-struct SimpleTrustRegion{T, CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
-    max_trust_radius::T
-    initial_trust_radius::T
-    step_threshold::T
-    shrink_threshold::T
-    expand_threshold::T
-    shrink_factor::T
-    expand_factor::T
-    max_shrink_times::Int
-    function SimpleTrustRegion(; chunk_size = Val{0}(),
-            autodiff = Val{true}(),
-            diff_type = Val{:forward},
-            max_trust_radius::Real = 0.0,
-            initial_trust_radius::Real = 0.0,
-            step_threshold::Real = 0.0001,
-            shrink_threshold::Real = 0.25,
-            expand_threshold::Real = 0.75,
-            shrink_factor::Real = 0.25,
-            expand_factor::Real = 2.0,
-            max_shrink_times::Int = 32)
-        new{typeof(initial_trust_radius),
-            SciMLBase._unwrap_val(chunk_size),
-            SciMLBase._unwrap_val(autodiff),
-            SciMLBase._unwrap_val(diff_type)}(max_trust_radius,
-            initial_trust_radius,
-            step_threshold,
-            shrink_threshold,
-            expand_threshold,
-            shrink_factor,
-            expand_factor,
-            max_shrink_times)
-    end
+@kwdef @concrete struct SimpleTrustRegion <: AbstractNewtonAlgorithm
+    autodiff = AutoForwardDiff()
+    max_trust_radius = 0.0
+    initial_trust_radius = 0.0
+    step_threshold = 0.0001
+    shrink_threshold = 0.25
+    expand_threshold = 0.75
+    shrink_factor = 0.25
+    expand_factor = 2.0
+    max_shrink_times::Int = 32
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem,
-        alg::SimpleTrustRegion, args...; abstol = nothing,
-        reltol = nothing,
-        maxiters = 1000, kwargs...)
-    f = Base.Fix2(prob.f, prob.p)
-    x = float(prob.u0)
-    T = typeof(x)
-    Δₘₐₓ = float(alg.max_trust_radius)
-    Δ = float(alg.initial_trust_radius)
-    η₁ = float(alg.step_threshold)
-    η₂ = float(alg.shrink_threshold)
-    η₃ = float(alg.expand_threshold)
-    t₁ = float(alg.shrink_factor)
-    t₂ = float(alg.expand_factor)
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        termination_condition = nothing, kwargs...)
+    @bb x = copy(float(prob.u0))
+    T = eltype(real(x))
+    Δₘₐₓ = T(alg.max_trust_radius)
+    Δ = T(alg.initial_trust_radius)
+    η₁ = T(alg.step_threshold)
+    η₂ = T(alg.shrink_threshold)
+    η₃ = T(alg.expand_threshold)
+    t₁ = T(alg.shrink_factor)
+    t₂ = T(alg.expand_factor)
     max_shrink_times = alg.max_shrink_times
 
-    if SciMLBase.isinplace(prob)
-        error("SimpleTrustRegion currently only supports out-of-place nonlinear problems")
-    end
+    fx = _get_fx(prob, x)
+    @bb xo = copy(x)
+    J, jac_cache = jacobian_cache(alg.autodiff, prob.f, fx, x, prob.p)
+    fx, ∇f = value_and_jacobian(alg.autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
-    atol = abstol !== nothing ? abstol :
-           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
-    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
-
-    if DiffEqBase.has_jac(prob.f)
-        ∇f = prob.f.jac(x, prob.p)
-        F = f(x)
-    elseif alg_autodiff(alg)
-        F, ∇f = value_derivative(f, x)
-    elseif x isa AbstractArray
-        F = f(x)
-        ∇f = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x), F)
-    else
-        F = f(x)
-        ∇f = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg), eltype(x), F)
-    end
+    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+        termination_condition)
 
     # Set default trust region radius if not specified by user.
-    if Δₘₐₓ == 0.0
-        Δₘₐₓ = max(norm(F), maximum(x) - minimum(x))
-    end
-    if Δ == 0.0
-        Δ = Δₘₐₓ / 11
-    end
+    Δₘₐₓ == 0 && (Δₘₐₓ = max(norm(fx), maximum(x) - minimum(x)))
+    Δ == 0 && (Δ = Δₘₐₓ / 11)
 
-    fₖ = 0.5 * norm(F)^2
+    fₖ = 0.5 * norm(fx)^2
     H = ∇f' * ∇f
-    g = ∇f' * F
+    g = ∇f' * fx
     shrink_counter = 0
 
+    @bb δsd = copy(x)
+    @bb δN_δsd = copy(x)
+    @bb δN = copy(x)
+    @bb Hδ = copy(x)
+    dogleg_cache = (; δsd, δN_δsd, δN)
+
+    F = fx
     for k in 1:maxiters
         # Solve the trust region subproblem.
-        δ = dogleg_method(∇f, F, g, Δ)
-        xₖ₊₁ = x + δ
-        Fₖ₊₁ = f(xₖ₊₁)
-        fₖ₊₁ = 0.5 * norm(Fₖ₊₁)^2
+        δ = dogleg_method!!(dogleg_cache, ∇f, fx, g, Δ)
+        @bb @. x = xo + δ
+
+        fx = __eval_f(prob, fx, x)
+
+        fₖ₊₁ = norm(fx)^2 / T(2)
 
         # Compute the ratio of the actual to predicted reduction.
-        model = -(δ' * g + 0.5 * δ' * H * δ)
-        r = (fₖ - fₖ₊₁) / model
+        @bb Hδ = H × δ
+        r = (fₖ₊₁ - fₖ) / (dot(δ', g) + dot(δ', Hδ) / T(2))
 
         # Update the trust region radius.
         if r < η₂
             Δ = t₁ * Δ
             shrink_counter += 1
-            if shrink_counter > max_shrink_times
-                return SciMLBase.build_solution(prob, alg, x, F;
-                    retcode = ReturnCode.Success)
-            end
+            shrink_counter > max_shrink_times && return build_solution(prob, alg, x, fx;
+                    retcode = ReturnCode.ConvergenceFailure)
         else
             shrink_counter = 0
         end
+
         if r > η₁
-            if isapprox(xₖ₊₁, x, atol = atol, rtol = rtol)
-                return SciMLBase.build_solution(prob, alg, xₖ₊₁, Fₖ₊₁;
-                    retcode = ReturnCode.Success)
-            end
+            # Termination Checks
+            tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+            tc_sol !== nothing && return tc_sol
+
             # Take the step.
-            x = xₖ₊₁
-            F = Fₖ₊₁
-            if alg_autodiff(alg)
-                F, ∇f = value_derivative(f, x)
-            elseif x isa AbstractArray
-                ∇f = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x),
-                    F)
-            else
-                ∇f = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg),
-                    eltype(x),
-                    F)
-            end
-
-            iszero(F) &&
-                return SciMLBase.build_solution(prob, alg, x, F;
-                    retcode = ReturnCode.Success)
+            @bb @. xo = x
+
+            fx, ∇f = value_and_jacobian(alg.autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
             # Update the trust region radius.
-            if r > η₃ && norm(δ) ≈ Δ
-                Δ = min(t₂ * Δ, Δₘₐₓ)
-            end
+            (r > η₃) && (norm(δ) ≈ Δ) && (Δ = min(t₂ * Δ, Δₘₐₓ))
             fₖ = fₖ₊₁
-            H = ∇f' * ∇f
-            g = ∇f' * F
+
+            @bb H = transpose(∇f) × ∇f
+            @bb g = transpose(∇f) × fx
         end
     end
-    return SciMLBase.build_solution(prob, alg, x, F; retcode = ReturnCode.MaxIters)
+
+    return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
+
+function dogleg_method!!(cache, J, f, g, Δ)
+    (; δsd, δN_δsd, δN) = cache
+
+    # Compute the Newton step.
+    @bb δN .= J \ f
+    @bb δN .*= -1
+    # Test if the full step is within the trust region.
+    (norm(δN) ≤ Δ) && return δN
+
+    # Calcualte Cauchy point, optimum along the steepest descent direction.
+    @bb δsd .= g
+    @bb @. δsd *= -1
+    norm_δsd = norm(δsd)
+    if (norm_δsd ≥ Δ)
+        @bb @. δsd *= Δ / norm_δsd
+        return δsd
+    end
+
+    # Find the intersection point on the boundary.
+    @bb @. δN_δsd = δN - δsd
+    dot_δN_δsd = dot(δN_δsd, δN_δsd)
+    dot_δsd_δN_δsd = dot(δsd, δN_δsd)
+    dot_δsd = dot(δsd, δsd)
+    fact = dot_δsd_δN_δsd^2 - dot_δN_δsd * (dot_δsd - Δ^2)
+    tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd
+    @bb @. δsd += tau * δN_δsd
+    return δsd
 end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 11caa7073..7b39fd6c7 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -170,31 +170,6 @@ function __init_identity_jacobian!!(J::StaticArray{S1, S2}) where {S1, S2}
         S1 * S2))
 end
 
-# function dogleg_method(J, f, g, Δ)
-#     # Compute the Newton step.
-#     δN = J \ (-f)
-#     # Test if the full step is within the trust region.
-#     if norm(δN) ≤ Δ
-#         return δN
-#     end
-
-#     # Calcualte Cauchy point, optimum along the steepest descent direction.
-#     δsd = -g
-#     norm_δsd = norm(δsd)
-#     if norm_δsd ≥ Δ
-#         return δsd .* Δ / norm_δsd
-#     end
-
-#     # Find the intersection point on the boundary.
-#     δN_δsd = δN - δsd
-#     dot_δN_δsd = dot(δN_δsd, δN_δsd)
-#     dot_δsd_δN_δsd = dot(δsd, δN_δsd)
-#     dot_δsd = dot(δsd, δsd)
-#     fact = dot_δsd_δN_δsd^2 - dot_δN_δsd * (dot_δsd - Δ^2)
-#     tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd
-#     return δsd + tau * δN_δsd
-# end
-
 @inline _vec(v) = vec(v)
 @inline _vec(v::Number) = v
 @inline _vec(v::AbstractVector) = v

From 82342fd64be1f18818a6c48043f68e7db89fca92 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 24 Nov 2023 14:58:00 -0500
Subject: [PATCH 231/375] Fix Limited Memory Broyden

---
 lib/SimpleNonlinearSolve/README.md            |   2 +
 .../src/SimpleNonlinearSolve.jl               |  48 ++---
 lib/SimpleNonlinearSolve/src/ad.jl            |  69 +++---
 .../src/nlsolve/lbroyden.jl                   | 199 ++++++++----------
 .../src/nlsolve/trustRegion.jl                |   9 +-
 lib/SimpleNonlinearSolve/src/utils.jl         |  27 ++-
 6 files changed, 167 insertions(+), 187 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index 0f52b1065..6bba38ff1 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -50,3 +50,5 @@ For more details on the bracketing methods, refer to the [Tutorials](https://doc
   - `Broyden` and `Klement` have been renamed to `SimpleBroyden` and `SimpleKlement` to
     avoid conflicts with `NonlinearSolve.jl`'s `GeneralBroyden` and `GeneralKlement`, which
     will be renamed to `Broyden` and `Klement` in the future.
+  - `LBroyden` has been renamed to `SimpleLimitedMemoryBroyden` to make it consistent with
+    `NonlinearSolve.jl`'s `LimitedMemoryBroyden`.
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 79a4d3f53..707f543a4 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -13,7 +13,7 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
     import ForwardDiff: Dual
     import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
     import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace
-    import StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray
+    import StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, MMatrix, Size
 end
 
 @reexport using ADTypes, SciMLBase
@@ -24,16 +24,16 @@ abstract type AbstractNewtonAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm e
 
 include("utils.jl")
 
-# Nonlinear Solvera
+## Nonlinear Solvers
 include("nlsolve/raphson.jl")
 include("nlsolve/broyden.jl")
-# include("nlsolve/lbroyden.jl")
+include("nlsolve/lbroyden.jl")
 include("nlsolve/klement.jl")
 include("nlsolve/trustRegion.jl")
 # include("nlsolve/halley.jl")
 # include("nlsolve/dfsane.jl")
 
-# Interval Nonlinear Solvers
+## Interval Nonlinear Solvers
 include("bracketing/bisection.jl")
 include("bracketing/falsi.jl")
 include("bracketing/ridder.jl")
@@ -42,7 +42,7 @@ include("bracketing/alefeld.jl")
 include("bracketing/itp.jl")
 
 # AD
-# include("ad.jl")
+include("ad.jl")
 
 ## Default algorithm
 
@@ -58,34 +58,22 @@ end
 
 @setup_workload begin
     for T in (Float32, Float64)
-        prob_no_brack = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-        algs = [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement(),
-            SimpleTrustRegion()]
-
-        @compile_workload begin
-            for alg in algs
-                solve(prob_no_brack, alg, abstol = T(1e-2))
-            end
-        end
+        prob_no_brack_scalar = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
+        prob_no_brack_iip = NonlinearProblem{true}((du, u, p) -> du .= u .* u .- p,
+            T.([1.0, 1.0, 1.0]), T(2))
+        prob_no_brack_oop = NonlinearProblem{false}((u, p) -> u .* u .- p,
+            T.([1.0, 1.0, 1.0]), T(2))
 
-        prob_no_brack = NonlinearProblem{true}((du, u, p) -> du .= u .* u .- p,
-            T.([1.0, 1.0]), T(2))
+        algs = [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement(),
+            SimpleTrustRegion(), SimpleLimitedMemoryBroyden(; threshold = 2)]
 
         @compile_workload begin
             for alg in algs
-                solve(prob_no_brack, alg, abstol = T(1e-2))
-            end
-        end
-
-        #=
-        for alg in (SimpleNewtonRaphson,)
-            for u0 in ([1., 1.], StaticArraysCore.SA[1.0, 1.0])
-                u0 = T.(.1)
-                probN = NonlinearProblem{false}((u,p) -> u .* u .- p, u0, T(2))
-                solve(probN, alg(), tol = T(1e-2))
+                solve(prob_no_brack_scalar, alg, abstol = T(1e-2))
+                solve(prob_no_brack_iip, alg, abstol = T(1e-2))
+                solve(prob_no_brack_oop, alg, abstol = T(1e-2))
             end
         end
-        =#
 
         prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
             T.((0.0, 2.0)), T(2))
@@ -98,9 +86,9 @@ end
     end
 end
 
-export SimpleBroyden,
-    SimpleGaussNewton, SimpleKlement, SimpleNewtonRaphson, SimpleTrustRegion
-# SimpleDFSane, SimpleHalley, LBroyden
+export SimpleBroyden, SimpleGaussNewton, SimpleKlement, SimpleLimitedMemoryBroyden,
+    SimpleNewtonRaphson, SimpleTrustRegion
+# SimpleDFSane, SimpleHalley
 export Alefeld, Bisection, Brent, Falsi, ITP, Ridder
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index b0fd9f11c..a13ae0e6f 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -1,7 +1,7 @@
 function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     f = prob.f
     p = value(prob.p)
-
+    u0 = value(prob.u0)
     if prob isa IntervalNonlinearProblem
         tspan = value(prob.tspan)
         newprob = IntervalNonlinearProblem(f, tspan, p; prob.kwargs...)
@@ -13,66 +13,57 @@ function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     sol = solve(newprob, alg, args...; kwargs...)
 
     uu = sol.u
-    if p isa Number
-        f_p = ForwardDiff.derivative(Base.Fix1(f, uu), p)
-    else
-        f_p = ForwardDiff.gradient(Base.Fix1(f, uu), p)
-    end
+    f_p = scalar_nlsolve_∂f_∂p(f, uu, p)
+    f_x = scalar_nlsolve_∂f_∂u(f, uu, p)
+
+    z_arr = -inv(f_x) * f_p
 
-    f_x = ForwardDiff.derivative(Base.Fix2(f, p), uu)
     pp = prob.p
-    sumfun = let f_x′ = -f_x
-        ((fp, p),) -> (fp / f_x′) * ForwardDiff.partials(p)
+    sumfun = ((z, p),) -> map(zᵢ -> zᵢ * ForwardDiff.partials(p), z)
+    if uu isa Number
+        partials = sum(sumfun, zip(z_arr, pp))
+    elseif p isa Number
+        partials = sumfun((z_arr, pp))
+    else
+        partials = sum(sumfun, zip(eachcol(z_arr), pp))
     end
-    partials = sum(sumfun, zip(f_p, pp))
+
     return sol, partials
 end
 
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
-            iip,
-            <:Dual{T, V, P}},
-        alg::AbstractSimpleNonlinearSolveAlgorithm,
-        args...; kwargs...) where {iip, T, V, P}
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, SVector, <:AbstractArray},
+            false, <:Dual{T, V, P}}, alg::AbstractSimpleNonlinearSolveAlgorithm, args...;
+        kwargs...) where {T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
-    return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
-        retcode = sol.retcode)
+    dual_soln = scalar_nlsolve_dual_soln(sol.u, partials, prob.p)
+    return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode)
 end
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, StaticArraysCore.SVector},
-            iip,
-            <:AbstractArray{<:Dual{T, V, P}}},
-        alg::AbstractSimpleNonlinearSolveAlgorithm, args...;
-        kwargs...) where {iip, T, V, P}
+
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, SVector, <:AbstractArray},
+            false, <:AbstractArray{<:Dual{T, V, P}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) where {T, V, P}
     sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
-    return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials), sol.resid;
-        retcode = sol.retcode)
+    dual_soln = scalar_nlsolve_dual_soln(sol.u, partials, prob.p)
+    return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode)
 end
 
 # avoid ambiguities
 for Alg in [Bisection]
     @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
-                <:Dual{T, V, P}},
-            alg::$Alg, args...;
-            kwargs...) where {uType, iip, T, V, P}
+                <:Dual{T, V, P}}, alg::$Alg, args...; kwargs...) where {uType, iip, T, V, P}
         sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
-        return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials),
-            sol.resid; retcode = sol.retcode,
+        dual_soln = scalar_nlsolve_dual_soln(sol.u, partials, prob.p)
+        return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
             left = Dual{T, V, P}(sol.left, partials),
             right = Dual{T, V, P}(sol.right, partials))
-        #return BracketingSolution(Dual{T,V,P}(sol.left, partials), Dual{T,V,P}(sol.right, partials), sol.retcode, sol.resid)
     end
     @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
-                <:AbstractArray{
-                    <:Dual{T,
-                        V,
-                        P},
-                }},
-            alg::$Alg, args...;
+                <:AbstractArray{<:Dual{T, V, P}}}, alg::$Alg, args...;
             kwargs...) where {uType, iip, T, V, P}
         sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
-        return SciMLBase.build_solution(prob, alg, Dual{T, V, P}(sol.u, partials),
-            sol.resid; retcode = sol.retcode,
+        dual_soln = scalar_nlsolve_dual_soln(sol.u, partials, prob.p)
+        return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
             left = Dual{T, V, P}(sol.left, partials),
             right = Dual{T, V, P}(sol.right, partials))
-        #return BracketingSolution(Dual{T,V,P}(sol.left, partials), Dual{T,V,P}(sol.right, partials), sol.retcode, sol.resid)
     end
 end
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 482092151..4cc8ee025 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -1,144 +1,119 @@
 """
-    LBroyden(; batched = false,
-              termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-                                                                  abstol = nothing, reltol = nothing),
-              threshold::Int = 27)
+    SimpleLimitedMemoryBroyden(; threshold::Int = 27)
+    SimpleLimitedMemoryBroyden(; threshold::Val = Val(27))
 
 A limited memory implementation of Broyden. This method applies the L-BFGS scheme to
-Broyden's method.
+Broyden's method. This Alogrithm unfortunately cannot non-allocating for StaticArrays
+without compromising on the "simple" aspect.
 
-!!! warn
+If the threshold is larger than the problem size, then this method will use `SimpleBroyden`.
 
-    This method is not very stable and can diverge even for very simple problems. This has mostly been
-    tested for neural networks in DeepEquilibriumNetworks.jl.
+!!! warning
+
+    This method is not very stable and can diverge even for very simple problems. This has
+    mostly been tested for neural networks in DeepEquilibriumNetworks.jl.
 """
-struct LBroyden{batched, TC <: NLSolveTerminationCondition} <:
-       AbstractSimpleNonlinearSolveAlgorithm
-    termination_condition::TC
-    threshold::Int
-
-    function LBroyden(; batched = false, threshold::Int = 27,
-            termination_condition = NLSolveTerminationCondition(NLSolveTerminationMode.NLSolveDefault;
-                abstol = nothing,
-                reltol = nothing))
-        return new{batched, typeof(termination_condition)}(termination_condition, threshold)
-    end
-end
+struct SimpleLimitedMemoryBroyden{threshold} <: AbstractSimpleNonlinearSolveAlgorithm end
 
-@views function SciMLBase.__solve(prob::NonlinearProblem, alg::LBroyden{batched}, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000,
-        kwargs...) where {batched}
-    tc = alg.termination_condition
-    mode = DiffEqBase.get_termination_mode(tc)
-    threshold = min(maxiters, alg.threshold)
-    x = float(prob.u0)
-
-    batched && @assert ndims(x)==2 "Batched LBroyden only supports 2D arrays"
-
-    if x isa Number
-        restore_scalar = true
-        x = [x]
-        f = u -> prob.f(u[], prob.p)
-    else
-        f = Base.Fix2(prob.f, prob.p)
-        restore_scalar = false
-    end
+__get_threshold(::SimpleLimitedMemoryBroyden{threshold}) where {threshold} = Val(threshold)
 
-    fₙ = f(x)
-    T = eltype(x)
+function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27))
+    return SimpleLimitedMemoryBroyden{SciMLBase._unwrap_val(threshold)}()
+end
 
-    if SciMLBase.isinplace(prob)
-        error("LBroyden currently only supports out-of-place nonlinear problems")
+@views function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
+        args...; abstol = nothing, reltol = nothing, maxiters = 1000,
+        termination_condition = nothing, kwargs...)
+    @bb x = copy(float(prob.u0))
+    threshold = __get_threshold(alg)
+    η = min(SciMLBase._unwrap_val(threshold), maxiters)
+
+    # For scalar problems / if the threshold is larger than problem size just use Broyden
+    if x isa Number || length(x) ≤ η
+        return SciMLBase.__solve(prob, SimpleBroyden(), args...;
+            abstol, reltol, maxiters, termination_condition, kwargs...)
     end
 
-    U, Vᵀ = _init_lbroyden_state(batched, x, threshold)
+    fx = _get_fx(prob, x)
 
-    atol = abstol !== nothing ? abstol :
-           (tc.abstol !== nothing ? tc.abstol :
-            real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5))
-    rtol = reltol !== nothing ? reltol :
-           (tc.reltol !== nothing ? tc.reltol : eps(real(one(eltype(T))))^(4 // 5))
+    U, Vᵀ = __init_low_rank_jacobian(x, fx, threshold)
 
-    if mode ∈ DiffEqBase.SAFE_BEST_TERMINATION_MODES
-        error("LBroyden currently doesn't support SAFE_BEST termination modes")
-    end
+    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+        termination_condition)
+
+    @bb xo = copy(x)
+    @bb δx = copy(fx)
+    @bb δx .*= -1
+    @bb fo = copy(fx)
+    @bb δf = copy(fx)
 
-    storage = mode ∈ DiffEqBase.SAFE_TERMINATION_MODES ? NLSolveSafeTerminationResult() :
-              nothing
-    termination_condition = tc(storage)
+    @bb vᵀ_cache = copy(x)
+    Tcache = __lbroyden_threshold_cache(x, threshold)
+    @bb mat_cache = copy(x)
 
-    xₙ = x
-    xₙ₋₁ = x
-    fₙ₋₁ = fₙ
-    update = fₙ
     for i in 1:maxiters
-        xₙ = xₙ₋₁ .+ update
-        fₙ = f(xₙ)
-        Δxₙ = xₙ .- xₙ₋₁
-        Δfₙ = fₙ .- fₙ₋₁
+        @bb @. x = xo + δx
+        fx = __eval_f(prob, fx, x)
+        @bb @. δf = fx - fo
 
-        if termination_condition(restore_scalar ? [fₙ] : fₙ, xₙ, xₙ₋₁, atol, rtol)
-            xₙ = restore_scalar ? xₙ[] : xₙ
-            return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.Success)
-        end
+        # Termination Checks
+        tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+        tc_sol !== nothing && return tc_sol
 
-        _U = selectdim(U, 1, 1:min(threshold, i))
-        _Vᵀ = selectdim(Vᵀ, 2, 1:min(threshold, i))
+        _U = selectdim(U, 2, 1:min(η, i - 1))
+        _Vᵀ = selectdim(Vᵀ, 1, 1:min(η, i - 1))
 
-        vᵀ = _rmatvec(_U, _Vᵀ, Δxₙ)
-        mvec = _matvec(_U, _Vᵀ, Δfₙ)
-        u = (Δxₙ .- mvec) ./ (sum(vᵀ .* Δfₙ) .+ convert(T, 1e-5))
+        vᵀ = _rmatvec!!(vᵀ_cache, Tcache, _U, _Vᵀ, δx)
+        mvec = _matvec!!(mat_cache, Tcache, _U, _Vᵀ, δf)
+        d = dot(vᵀ, δf)
+        @bb @. δx = (δx - mvec) / d
 
-        selectdim(Vᵀ, 2, mod1(i, threshold)) .= vᵀ
-        selectdim(U, 1, mod1(i, threshold)) .= u
+        selectdim(U, 2, mod1(i, η)) .= δx
+        selectdim(Vᵀ, 1, mod1(i, η)) .= vᵀ
 
-        update = -_matvec(selectdim(U, 1, 1:min(threshold, i + 1)),
-            selectdim(Vᵀ, 2, 1:min(threshold, i + 1)), fₙ)
+        _U = selectdim(U, 2, 1:min(η, i))
+        _Vᵀ = selectdim(Vᵀ, 1, 1:min(η, i))
+        δx = _matvec!!(δx, Tcache, _U, _Vᵀ, fx)
+        @bb @. δx *= -1
 
-        xₙ₋₁ = xₙ
-        fₙ₋₁ = fₙ
+        @bb copyto!(xo, x)
+        @bb copyto!(fo, fx)
     end
 
-    xₙ = restore_scalar ? xₙ[] : xₙ
-    return SciMLBase.build_solution(prob, alg, xₙ, fₙ; retcode = ReturnCode.MaxIters)
+    return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end
 
-function _init_lbroyden_state(batched::Bool, x, threshold)
-    T = eltype(x)
-    if batched
-        U = fill!(similar(x, (threshold, size(x, 1), size(x, 2))), zero(T))
-        Vᵀ = fill!(similar(x, (size(x, 1), threshold, size(x, 2))), zero(T))
-    else
-        U = fill!(similar(x, (threshold, length(x))), zero(T))
-        Vᵀ = fill!(similar(x, (length(x), threshold)), zero(T))
+function _rmatvec!!(y, xᵀU, U, Vᵀ, x)
+    # xᵀ × (-I + UVᵀ)
+    η = size(U, 2)
+    if η == 0
+        @bb @. y = -x
+        return y
     end
-    return U, Vᵀ
+    x_ = vec(x)
+    xᵀU_ = view(xᵀU, 1:η)
+    @bb xᵀU_ = transpose(U) × x_
+    @bb y = transpose(Vᵀ) × xᵀU_
+    @bb @. y -= x
+    return y
 end
 
-function _rmatvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
-        x::Union{<:AbstractVector, <:Number})
-    length(U) == 0 && return x
-    return -x .+ vec((x' * Vᵀ) * U)
-end
-
-function _rmatvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
-        x::AbstractMatrix) where {T1, T2}
-    length(U) == 0 && return x
-    Vᵀx = sum(Vᵀ .* reshape(x, size(x, 1), 1, size(x, 2)); dims = 1)
-    return -x .+ _drdims_sum(U .* permutedims(Vᵀx, (2, 1, 3)); dims = 1)
-end
-
-function _matvec(U::AbstractMatrix, Vᵀ::AbstractMatrix,
-        x::Union{<:AbstractVector, <:Number})
-    length(U) == 0 && return x
-    return -x .+ vec(Vᵀ * (U * x))
+function _matvec!!(y, Vᵀx, U, Vᵀ, x)
+    # (-I + UVᵀ) × x
+    η = size(U, 2)
+    if η == 0
+        @bb @. y = -x
+        return y
+    end
+    x_ = vec(x)
+    Vᵀx_ = view(Vᵀx, 1:η)
+    @bb Vᵀx_ = Vᵀ × x_
+    @bb y = U × Vᵀx_
+    @bb @. y -= x
+    return y
 end
 
-function _matvec(U::AbstractArray{T1, 3}, Vᵀ::AbstractArray{T2, 3},
-        x::AbstractMatrix) where {T1, T2}
-    length(U) == 0 && return x
-    xUᵀ = sum(reshape(x, size(x, 1), 1, size(x, 2)) .* permutedims(U, (2, 1, 3)); dims = 1)
-    return -x .+ _drdims_sum(xUᵀ .* Vᵀ; dims = 2)
+__lbroyden_threshold_cache(x, ::Val{threshold}) where {threshold} = similar(x, threshold)
+function __lbroyden_threshold_cache(x::SArray, ::Val{threshold}) where {threshold}
+    return SArray{Tuple{threshold}, eltype(x)}(ntuple(_ -> zero(eltype(x)), threshold))
 end
-
-_drdims_sum(args...; dims = :) = dropdims(sum(args...; dims); dims)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 2420b7226..3c3ad60b4 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -1,9 +1,8 @@
 """
     SimpleTrustRegion(; autodiff = AutoForwardDiff(), max_trust_radius::Real = 0.0,
-                        initial_trust_radius::Real = 0.0, step_threshold::Real = 0.1,
-                        shrink_threshold::Real = 0.25, expand_threshold::Real = 0.75,
-                        shrink_factor::Real = 0.25, expand_factor::Real = 2.0,
-                        max_shrink_times::Int = 32)
+        initial_trust_radius::Real = 0.0, step_threshold::Real = 0.1,
+        shrink_threshold::Real = 0.25, expand_threshold::Real = 0.75,
+        shrink_factor::Real = 0.25, expand_factor::Real = 2.0, max_shrink_times::Int = 32)
 
 A low-overhead implementation of a trust-region solver. This method is non-allocating on
 scalar and static array problems.
@@ -105,7 +104,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
             Δ = t₁ * Δ
             shrink_counter += 1
             shrink_counter > max_shrink_times && return build_solution(prob, alg, x, fx;
-                    retcode = ReturnCode.ConvergenceFailure)
+                retcode = ReturnCode.ConvergenceFailure)
         else
             shrink_counter = 0
         end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 7b39fd6c7..396a134da 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -170,6 +170,20 @@ function __init_identity_jacobian!!(J::StaticArray{S1, S2}) where {S1, S2}
         S1 * S2))
 end
 
+function __init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2},
+        ::Val{threshold}) where {S1, S2, T1, T2, threshold}
+    T = promote_type(T1, T2)
+    fuSize, uSize = Size(fu), Size(u)
+    Vᵀ = MArray{Tuple{threshold, prod(uSize)}, T}(undef)
+    U = MArray{Tuple{prod(fuSize), threshold}, T}(undef)
+    return U, Vᵀ
+end
+function __init_low_rank_jacobian(u, fu, ::Val{threshold}) where {threshold}
+    Vᵀ = similar(u, threshold, length(u))
+    U = similar(u, length(fu), threshold)
+    return U, Vᵀ
+end
+
 @inline _vec(v) = vec(v)
 @inline _vec(v::Number) = v
 @inline _vec(v::AbstractVector) = v
@@ -200,10 +214,17 @@ end
 
 # Termination Conditions Support
 # Taken directly from NonlinearSolve.jl
+# The default here is different from NonlinearSolve since the userbases are assumed to be
+# different. NonlinearSolve is more for robust / cached solvers while SimpleNonlinearSolve
+# is meant for low overhead solvers, users can opt into the other termination modes but the
+# default is to use the least overhead version.
 function init_termination_cache(abstol, reltol, du, u, ::Nothing)
-    return init_termination_cache(abstol, reltol, du, u, AbsSafeBestTerminationMode())
+    return init_termination_cache(abstol, reltol, du, u, AbsNormTerminationMode())
 end
 function init_termination_cache(abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode)
+    T = promote_type(eltype(du), eltype(u))
+    abstol !== nothing && (abstol = T(abstol))
+    reltol !== nothing && (reltol = T(reltol))
     tc_cache = init(du, u, tc; abstol, reltol)
     return DiffEqBase.get_abstol(tc_cache), DiffEqBase.get_reltol(tc_cache), tc_cache
 end
@@ -257,5 +278,9 @@ function check_termination(tc_cache, fx, x, xo, prob, alg,
     return nothing
 end
 
+@inline value(x) = x
+@inline value(x::Dual) = ForwardDiff.value(x)
+@inline value(x::AbstractArray{<:Dual}) = map(ForwardDiff.value, x)
+
 @inline __eval_f(prob, fx, x) = isinplace(prob) ? (prob.f(fx, x, prob.p); fx) :
                                 prob.f(x, prob.p)

From 0c2b3658e59c2a62218f4650d98346db50fd5be2 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 24 Nov 2023 15:50:33 -0500
Subject: [PATCH 232/375] Type stability fixes

---
 lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl | 4 ++--
 lib/SimpleNonlinearSolve/src/utils.jl           | 6 ++----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
index 3d8debfe4..22f7fba84 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
@@ -47,8 +47,8 @@ function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresPr
         end
 
         @bb copyto!(xo, x)
-        Δx = _restructure(x, dfx \ _vec(fx))
-        @bb x .-= Δx
+        δx = _restructure(x, dfx \ _vec(fx))
+        @bb x .-= δx
     end
 
     return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 396a134da..464495500 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -161,13 +161,11 @@ function __init_identity_jacobian!!(J)
 end
 function __init_identity_jacobian(u::StaticArray, fu)
     S1, S2 = length(fu), length(u)
-    J = SMatrix{S1, S2, eltype(u)}(ntuple(i -> ifelse(i ∈ 1:(S1 + 1):(S1 * S2), 1, 0),
-        S1 * S2))
+    J = SMatrix{S1, S2, eltype(u)}(I)
     return J
 end
 function __init_identity_jacobian!!(J::StaticArray{S1, S2}) where {S1, S2}
-    return SMMatrix{S1, S2, eltype(J)}(ntuple(i -> ifelse(i ∈ 1:(S1 + 1):(S1 * S2), 1, 0),
-        S1 * S2))
+    return SMMatrix{S1, S2, eltype(J)}(I)
 end
 
 function __init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2},

From 8f6d66db21219c233f1e0c1910275720c59b8335 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 24 Nov 2023 22:24:14 -0500
Subject: [PATCH 233/375] Fix Halley's method

---
 .../src/SimpleNonlinearSolve.jl               |  11 +-
 .../src/nlsolve/dfsane.jl                     | 198 ++++++++----------
 .../src/nlsolve/halley.jl                     | 143 +++++--------
 lib/SimpleNonlinearSolve/src/utils.jl         |  43 ++++
 4 files changed, 186 insertions(+), 209 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 707f543a4..a9e7d7b82 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -30,8 +30,8 @@ include("nlsolve/broyden.jl")
 include("nlsolve/lbroyden.jl")
 include("nlsolve/klement.jl")
 include("nlsolve/trustRegion.jl")
-# include("nlsolve/halley.jl")
-# include("nlsolve/dfsane.jl")
+include("nlsolve/halley.jl")
+include("nlsolve/dfsane.jl")
 
 ## Interval Nonlinear Solvers
 include("bracketing/bisection.jl")
@@ -64,7 +64,7 @@ end
         prob_no_brack_oop = NonlinearProblem{false}((u, p) -> u .* u .- p,
             T.([1.0, 1.0, 1.0]), T(2))
 
-        algs = [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement(),
+        algs = [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
             SimpleTrustRegion(), SimpleLimitedMemoryBroyden(; threshold = 2)]
 
         @compile_workload begin
@@ -86,9 +86,8 @@ end
     end
 end
 
-export SimpleBroyden, SimpleGaussNewton, SimpleKlement, SimpleLimitedMemoryBroyden,
-    SimpleNewtonRaphson, SimpleTrustRegion
-# SimpleDFSane, SimpleHalley
+export SimpleBroyden, SimpleDFSane, SimpleGaussNewton, SimpleHalley, SimpleKlement,
+    SimpleLimitedMemoryBroyden, SimpleNewtonRaphson, SimpleTrustRegion
 export Alefeld, Bisection, Brent, Falsi, ITP, Ridder
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 0ecc545f6..657f760cb 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -53,117 +53,91 @@ end
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         termination_condition = nothing, kwargs...)
+    x = float(copy(prob.u0))
+    fx = _get_fx(prob, x)
+    T = eltype(x)
 
-    # f = isinplace(prob) ? (du, u) -> prob.f(du, u, prob.p) : u -> prob.f(u, prob.p)
-
-    # x = float(prob.u0)
-    # fx = _get_fx(prob, x)
-    # T = eltype(x)
-
-    # σ_min = T(alg.σ_min)
-    # σ_max = T(alg.σ_max)
-    # σ_k = T(alg.σ_1)
-
-    # M = alg.M
-    # γ = T(alg.γ)
-    # τ_min = T(alg.τ_min)
-    # τ_max = T(alg.τ_max)
-    # nexp = alg.nexp
-    # η_strategy = alg.η_strategy
-
-    # abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
-    #     termination_condition)
-
-    # ff = if isinplace(prob)
-    #     function (_fx, x)
-    #         f(_fx, x)
-    #         f_k = norm(_fx)^nexp
-    #         return f_k, _fx
-    #     end
-    # else
-    #     function (x)
-    #         _fx = f(x)
-    #         f_k = norm(_fx)^nexp
-    #         return f_k, _fx
-    #     end
-    # end
-
-    # generate_history(f_k, M) = fill(f_k, M)
-
-    # f_k, F_k = isinplace(prob) ? ff(fx, x) : ff(x)
-    # F_k = __copy(F_k)
-    # α_1 = one(T)
-    # f_1 = f_k
-    # history_f_k = generate_history(f_k, M)
-
-    # # Generate the cache
-    # d, xo, x_cache, δx, δf = __copy(x), __copy(x), __copy(x), __copy(x), __copy(x)
-    # α_tp, α_tm = __copy(x), __copy(x)
-
-    # for k in 1:maxiters
-    #     # Spectral parameter range check
-    #     σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
-
-    #     # Line search direction
-    #     d = __broadcast!!(d, *, -σ_k, F_k)
-
-    #     η = η_strategy(f_1, k, x, F_k)
-    #     f̄ = maximum(history_f_k)
-    #     α_p = α_1
-    #     α_m = α_1
-
-    #     x_cache = __broadcast!!(x_cache, *, α_p, d)
-    #     x = __broadcast!!(x, +, x_cache)
-
-    #     f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
-
-    #     # FIXME: This part is not correctly implemented
-    #     while true
-    #         criteria = f̄ + η - γ * α_p^2 * f_k
-    #         f_new ≤ criteria && break
-
-    #         if ArrayInterface.can_setindex(α_tp) && !(x isa Number)
-    #             @. α_tp = α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
-    #         else
-    #             α_tp = @. α_p^2 * f_k / (f_new + (2 * α_p - 1) * f_k)
-    #         end
-    #         x_cache = __broadcast!!(x_cache, *, α_m, d)
-    #         x = __broadcast!!(x, -, x_cache)
-    #         f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
-
-    #         f_new ≤ criteria && break
-
-    #         if ArrayInterface.can_setindex(α_tm) && !(x isa Number)
-    #             @. α_tm = α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
-    #             @. α_p = clamp(α_tp, τ_min * α_p, τ_max * α_p)
-    #             @. α_m = clamp(α_tm, τ_min * α_m, τ_max * α_m)
-    #         else
-    #             α_tm = @. α_m^2 * f_k / (f_new + (2 * α_m - 1) * f_k)
-    #             α_p = @. clamp(α_tp, τ_min * α_p, τ_max * α_p)
-    #             α_m = @. clamp(α_tm, τ_min * α_m, τ_max * α_m)
-    #         end
-    #         x_cache = __broadcast!!(x_cache, *, α_p, d)
-    #         x = __broadcast!!(x, +, x_cache)
-    #         f_new, F_new = isinplace(prob) ? ff(fx, x) : ff(x)
-    #     end
-
-    #     tc_sol = check_termination(tc_cache, f_new, x, xo, prob, alg)
-    #     tc_sol !== nothing && return tc_sol
-
-    #     # Update spectral parameter
-    #     δx = __broadcast!!(δx, -, x, xo)
-    #     δf = __broadcast!!(δf, -, F_new, F_k)
-
-    #     σ_k = dot(δx, δx) / dot(δx, δf)
-
-    #     # Take step
-    #     xo = __copyto!!(xo, x)
-    #     F_k = __copyto!!(F_k, F_new)
-    #     f_k = f_new
-
-    #     # Store function value
-    #     history_f_k[k % M + 1] = f_new
-    # end
-
-    # return build_solution(prob, alg, x, F_k; retcode = ReturnCode.MaxIters)
+    σ_min = T(alg.σ_min)
+    σ_max = T(alg.σ_max)
+    σ_k = T(alg.σ_1)
+
+    (; M, nexp, η_strategy) = alg
+    γ = T(alg.γ)
+    τ_min = T(alg.τ_min)
+    τ_max = T(alg.τ_max)
+
+    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+        termination_condition)
+
+    fx_norm = norm(fx)^nexp
+    α_1 = one(T)
+    f_1 = fx_norm
+    history_f_k = fill(fx_norm, M)
+
+    # Generate the cache
+    @bb d = copy(x)
+    @bb xo = copy(x)
+    @bb x_cache = copy(x)
+    @bb δx = copy(x)
+    @bb fxo = copy(fx)
+    @bb δf = copy(fx)
+
+    k = 0
+    while k < maxiters
+        # Spectral parameter range check
+        σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
+
+        # Line search direction
+        @bb @. d = -σ_k * fx
+
+        η = η_strategy(f_1, k, x, fx)
+        f_bar = maximum(history_f_k)
+        α_p = α_1
+        α_m = α_1
+
+        @bb @. x += α_p * d
+
+        fx = __eval_f(prob, fx, x)
+        fx_norm_new = norm(fx)^nexp
+
+        while k < maxiters
+            fx_norm_new ≤ (f_bar + η - γ * α_p^2 * fx_norm) && break
+
+            α_p = α_p^2 * fx_norm / (fx_norm_new + (T(2) * α_p - T(1)) * fx_norm)
+            @bb @. x -= α_m * d
+
+            fx = __eval_f(prob, fx, x)
+            fx_norm_new = norm(fx)^nexp
+
+            fx_norm_new ≤ (f_bar + η - γ * α_m^2 * fx_norm) && break
+
+            α_tm = α_m^2 * fx_norm / (fx_norm_new + (T(2) * α_m - T(1)) * fx_norm)
+            α_p = clamp(α_p, τ_min * α_p, τ_max * α_p)
+            α_m = clamp(α_tm, τ_min * α_m, τ_max * α_m)
+            @bb @. x += α_p * d
+
+            fx = __eval_f(prob, fx, x)
+            fx_norm_new = norm(fx)^nexp
+        end
+
+        tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+        tc_sol !== nothing && return tc_sol
+
+        # Update spectral parameter
+        @bb @. δx = x - xo
+        @bb @. δf = fx - fxo
+
+        σ_k = dot(δx, δx) / dot(δx, δf)
+
+        # Take step
+        @bb copyto!(xo, x)
+        @bb copyto!(fxo, fx)
+        fx_norm = fx_norm_new
+
+        # Store function value
+        history_f_k[mod1(k, M)] = fx_norm_new
+        k += 1
+    end
+
+    return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 8131acada..3e6e4d55f 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -1,11 +1,8 @@
 """
-```julia
-SimpleHalley(; chunk_size = Val{0}(), autodiff = Val{true}(),
-    diff_type = Val{:forward})
-```
+    SimpleHalley(autodiff)
+    SimpleHalley(; autodiff = AutoForwardDiff())
 
-A low-overhead implementation of SimpleHalley's Method. This method is non-allocating on scalar
-and static array problems.
+A low-overhead implementation of Halley's Method.
 
 !!! note
 
@@ -15,104 +12,68 @@ and static array problems.
 
 ### Keyword Arguments
 
-  - `chunk_size`: the chunk size used by the internal ForwardDiff.jl automatic differentiation
-    system. This allows for multiple derivative columns to be computed simultaneously,
-    improving performance. Defaults to `0`, which is equivalent to using ForwardDiff.jl's
-    default chunk size mechanism. For more details, see the documentation for
-    [ForwardDiff.jl](https://juliadiff.org/ForwardDiff.jl/stable/).
-  - `autodiff`: whether to use forward-mode automatic differentiation for the Jacobian.
-    Note that this argument is ignored if an analytical Jacobian is passed; as that will be
-    used instead. Defaults to `Val{true}`, which means ForwardDiff.jl is used by default.
-    If `Val{false}`, then FiniteDiff.jl is used for finite differencing.
-  - `diff_type`: the type of finite differencing used if `autodiff = false`. Defaults to
-    `Val{:forward}` for forward finite differences. For more details on the choices, see the
-    [FiniteDiff.jl](https://github.com/JuliaDiff/FiniteDiff.jl) documentation.
+  - `autodiff`: determines the backend used for the Hessian. Defaults to
+    `AutoForwardDiff()`. Valid choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
 """
-struct SimpleHalley{CS, AD, FDT} <: AbstractNewtonAlgorithm{CS, AD, FDT}
-    function SimpleHalley(; chunk_size = Val{0}(), autodiff = Val{true}(),
-            diff_type = Val{:forward})
-        new{SciMLBase._unwrap_val(chunk_size), SciMLBase._unwrap_val(autodiff),
-            SciMLBase._unwrap_val(diff_type)}()
-    end
+@kwdef @concrete struct SimpleHalley <: AbstractNewtonAlgorithm
+    autodiff = AutoForwardDiff()
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem,
-        alg::SimpleHalley, args...; abstol = nothing,
-        reltol = nothing,
-        maxiters = 1000, kwargs...)
-    f = Base.Fix2(prob.f, prob.p)
-    x = float(prob.u0)
-    fx = f(x)
-    if isa(x, AbstractArray)
-        n = length(x)
-    end
-    T = typeof(x)
-
-    if SciMLBase.isinplace(prob)
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        termination_condition = nothing, kwargs...)
+    isinplace(prob) &&
         error("SimpleHalley currently only supports out-of-place nonlinear problems")
-    end
 
-    atol = abstol !== nothing ? abstol :
-           real(oneunit(eltype(T))) * (eps(real(one(eltype(T)))))^(4 // 5)
-    rtol = reltol !== nothing ? reltol : eps(real(one(eltype(T))))^(4 // 5)
+    x = copy(float(prob.u0))
+    fx = _get_fx(prob, x)
+    T = eltype(x)
 
-    if x isa Number
-        xo = oftype(one(eltype(x)), Inf)
+    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+        termination_condition)
+
+    @bb xo = copy(x)
+
+    if setindex_trait(x) === CanSetindex()
+        A = similar(x, length(x), length(x))
+        Aaᵢ = similar(x, length(x))
+        cᵢ = similar(x)
     else
-        xo = map(x -> oftype(one(eltype(x)), Inf), x)
+        A = x
+        Aaᵢ = x
+        cᵢ = x
     end
 
     for i in 1:maxiters
-        if alg_autodiff(alg)
-            if isa(x, Number)
-                fx = f(x)
-                dfx = ForwardDiff.derivative(f, x)
-                d2fx = ForwardDiff.derivative(x -> ForwardDiff.derivative(f, x), x)
-            else
-                fx = f(x)
-                dfx = ForwardDiff.jacobian(f, x)
-                d2fx = ForwardDiff.jacobian(x -> ForwardDiff.jacobian(f, x), x)
-                ai = -(dfx \ fx)
-                A = reshape(d2fx * ai, (n, n))
-                bi = (dfx) \ (A * ai)
-                ci = (ai .* ai) ./ (ai .+ (0.5 .* bi))
-            end
-        else
-            if isa(x, Number)
-                fx = f(x)
-                dfx = FiniteDiff.finite_difference_derivative(f, x, diff_type(alg),
-                    eltype(x))
-                d2fx = FiniteDiff.finite_difference_derivative(x -> FiniteDiff.finite_difference_derivative(f,
-                        x),
-                    x,
-                    diff_type(alg), eltype(x))
-            else
-                fx = f(x)
-                dfx = FiniteDiff.finite_difference_jacobian(f, x, diff_type(alg), eltype(x))
-                d2fx = FiniteDiff.finite_difference_jacobian(x -> FiniteDiff.finite_difference_jacobian(f,
-                        x),
-                    x,
-                    diff_type(alg), eltype(x))
-                ai = -(dfx \ fx)
-                A = reshape(d2fx * ai, (n, n))
-                bi = (dfx) \ (A * ai)
-                ci = (ai .* ai) ./ (ai .+ (0.5 .* bi))
+        # Hessian Computation is unfortunately type unstable
+        fx, dfx, d2fx = compute_jacobian_and_hessian(alg.autodiff, prob, fx, x)
+        setindex_trait(x) === CannotSetindex() && (A = dfx)
+
+        aᵢ = dfx \ _vec(fx)
+        A_ = _vec(A)
+        @bb A_ = d2fx × aᵢ
+        A = _restructure(A, A_)
+
+        @bb Aaᵢ = A × aᵢ
+        @bb A .*= -1
+        bᵢ = dfx \ Aaᵢ
+
+        @bb @. cᵢ = (aᵢ * aᵢ) / (-aᵢ + (T(0.5) * bᵢ))
+
+        if i == 1
+            if iszero(fx)
+                return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
             end
-        end
-        iszero(fx) &&
-            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-        if isa(x, Number)
-            Δx = (2 * dfx^2 - fx * d2fx) \ (2fx * dfx)
-            x -= Δx
         else
-            Δx = ci
-            x += Δx
-        end
-        if isapprox(x, xo, atol = atol, rtol = rtol)
-            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+            # Termination Checks
+            tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+            tc_sol !== nothing && return tc_sol
         end
-        xo = x
+
+        @bb @. x += cᵢ
+
+        @bb copyto!(xo, x)
     end
 
-    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+    return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 464495500..7dbd8e422 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -146,6 +146,49 @@ end
 
 jacobian_cache(ad, f::F, y, x::Number, p) where {F} = nothing, nothing
 
+function compute_jacobian_and_hessian(ad::AutoForwardDiff, prob, _, x::Number)
+    fx = prob.f(x, prob.p)
+    J_fn = Base.Fix1(ForwardDiff.derivative, Base.Fix2(prob.f, prob.p))
+    dfx = J_fn(x)
+    d2fx = ForwardDiff.derivative(J_fn, x)
+    return fx, dfx, d2fx
+end
+
+function compute_jacobian_and_hessian(ad::AutoForwardDiff, prob, fx, x)
+    if isinplace(prob)
+        error("Inplace version for Nested ForwardDiff Not Implemented Yet!")
+    else
+        f = Base.Fix2(prob.f, prob.p)
+        fx = f(x)
+        J_fn = Base.Fix1(ForwardDiff.jacobian, f)
+        dfx = J_fn(x)
+        d2fx = ForwardDiff.jacobian(J_fn, x)
+        return fx, dfx, d2fx
+    end
+end
+
+function compute_jacobian_and_hessian(ad::AutoFiniteDiff, prob, _, x::Number)
+    fx = prob.f(x, prob.p)
+    J_fn = x -> FiniteDiff.finite_difference_derivative(Base.Fix2(prob.f, prob.p), x,
+        ad.fdtype)
+    dfx = J_fn(x)
+    d2fx = FiniteDiff.finite_difference_derivative(J_fn, x, ad.fdtype)
+    return fx, dfx, d2fx
+end
+
+function compute_jacobian_and_hessian(ad::AutoFiniteDiff, prob, fx, x)
+    if isinplace(prob)
+        error("Inplace version for Nested FiniteDiff Not Implemented Yet!")
+    else
+        f = Base.Fix2(prob.f, prob.p)
+        fx = f(x)
+        J_fn = x -> FiniteDiff.finite_difference_jacobian(f, x, ad.fdtype)
+        dfx = J_fn(x)
+        d2fx = FiniteDiff.finite_difference_jacobian(J_fn, x, ad.fdtype)
+        return fx, dfx, d2fx
+    end
+end
+
 __init_identity_jacobian(u::Number, _) = one(u)
 __init_identity_jacobian!!(J::Number) = one(J)
 function __init_identity_jacobian(u, fu)

From 83102b059b9a1778c3525a883e2738c89e273683 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 24 Nov 2023 23:32:50 -0500
Subject: [PATCH 234/375] Add tests for the nonlinear solvers

---
 .../src/SimpleNonlinearSolve.jl               |   7 +
 lib/SimpleNonlinearSolve/src/ad.jl            |  21 +
 lib/SimpleNonlinearSolve/src/utils.jl         |   4 +-
 lib/SimpleNonlinearSolve/test/Project.toml    |   3 +-
 lib/SimpleNonlinearSolve/test/basictests.jl   | 948 ++++++++----------
 lib/SimpleNonlinearSolve/test/inplace.jl      |  52 -
 .../test/least_squares.jl                     |   8 +-
 lib/SimpleNonlinearSolve/test/runtests.jl     |   1 -
 8 files changed, 445 insertions(+), 599 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/test/inplace.jl

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index a9e7d7b82..66d7d4271 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -67,12 +67,19 @@ end
         algs = [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
             SimpleTrustRegion(), SimpleLimitedMemoryBroyden(; threshold = 2)]
 
+        algs_no_iip = [SimpleHalley()]
+
         @compile_workload begin
             for alg in algs
                 solve(prob_no_brack_scalar, alg, abstol = T(1e-2))
                 solve(prob_no_brack_iip, alg, abstol = T(1e-2))
                 solve(prob_no_brack_oop, alg, abstol = T(1e-2))
             end
+
+            for alg in algs_no_iip
+                solve(prob_no_brack_scalar, alg, abstol = T(1e-2))
+                solve(prob_no_brack_oop, alg, abstol = T(1e-2))
+            end
         end
 
         prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index a13ae0e6f..8cbff710c 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -47,6 +47,27 @@ function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, SVector, <:Abstr
     return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode)
 end
 
+function scalar_nlsolve_∂f_∂p(f, u, p)
+    ff = p isa Number ? ForwardDiff.derivative :
+         (u isa Number ? ForwardDiff.gradient : ForwardDiff.jacobian)
+    return ff(Base.Fix1(f, u), p)
+end
+
+function scalar_nlsolve_∂f_∂u(f, u, p)
+    ff = u isa Number ? ForwardDiff.derivative : ForwardDiff.jacobian
+    return ff(Base.Fix2(f, p), u)
+end
+
+function scalar_nlsolve_dual_soln(u::Number, partials,
+        ::Union{<:AbstractArray{<:Dual{T, V, P}}, Dual{T, V, P}}) where {T, V, P}
+    return Dual{T, V, P}(u, partials)
+end
+
+function scalar_nlsolve_dual_soln(u::AbstractArray, partials,
+        ::Union{<:AbstractArray{<:Dual{T, V, P}}, Dual{T, V, P}}) where {T, V, P}
+    return map(((uᵢ, pᵢ),) -> Dual{T, V, P}(uᵢ, pᵢ), zip(u, partials))
+end
+
 # avoid ambiguities
 for Alg in [Bisection]
     @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 7dbd8e422..870b526f4 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -207,8 +207,8 @@ function __init_identity_jacobian(u::StaticArray, fu)
     J = SMatrix{S1, S2, eltype(u)}(I)
     return J
 end
-function __init_identity_jacobian!!(J::StaticArray{S1, S2}) where {S1, S2}
-    return SMMatrix{S1, S2, eltype(J)}(I)
+function __init_identity_jacobian!!(J::SMatrix{S1, S2}) where {S1, S2}
+    return SMatrix{S1, S2, eltype(J)}(I)
 end
 
 function __init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2},
diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
index 469f302ee..835a6aa43 100644
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -3,8 +3,9 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 027f766ed..4963a52fe 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,586 +1,454 @@
-using SimpleNonlinearSolve,
-    StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
-    NNlib
-
-const BATCHED_BROYDEN_SOLVERS = []
-const BROYDEN_SOLVERS = []
-const BATCHED_LBROYDEN_SOLVERS = []
-const LBROYDEN_SOLVERS = []
-const BATCHED_DFSANE_SOLVERS = []
-const DFSANE_SOLVERS = []
-const BATCHED_RAPHSON_SOLVERS = []
-
-for mode in instances(NLSolveTerminationMode.T)
-    if mode ∈
-       (NLSolveTerminationMode.SteadyStateDefault, NLSolveTerminationMode.RelSafeBest,
-        NLSolveTerminationMode.AbsSafeBest)
-        continue
-    end
-
-    termination_condition = NLSolveTerminationCondition(mode; abstol = nothing,
-        reltol = nothing)
-    push!(BROYDEN_SOLVERS, Broyden(; batched = false, termination_condition))
-    push!(BATCHED_BROYDEN_SOLVERS, Broyden(; batched = true, termination_condition))
-    push!(LBROYDEN_SOLVERS, LBroyden(; batched = false, termination_condition))
-    push!(BATCHED_LBROYDEN_SOLVERS, LBroyden(; batched = true, termination_condition))
-    push!(DFSANE_SOLVERS, SimpleDFSane(; batched = false, termination_condition))
-    push!(BATCHED_DFSANE_SOLVERS, SimpleDFSane(; batched = true, termination_condition))
-    push!(BATCHED_RAPHSON_SOLVERS,
-        SimpleNewtonRaphson(; batched = true,
-            termination_condition))
-    push!(BATCHED_RAPHSON_SOLVERS,
-        SimpleNewtonRaphson(; batched = true, autodiff = false,
-            termination_condition))
-end
-
-# SimpleNewtonRaphson
-function benchmark_scalar(f, u0)
-    probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, SimpleNewtonRaphson()))
-end
-
-function ff(u, p)
-    u .* u .- 2
-end
-const cu0 = @SVector[1.0, 1.0]
-function sf(u, p)
-    u * u - 2
-end
-const csu0 = 1.0
-
-sol = benchmark_scalar(sf, csu0)
-@test sol.retcode === ReturnCode.Success
-@test sol.u * sol.u - 2 < 1e-9
-
-if VERSION >= v"1.7"
-    @test (@ballocated benchmark_scalar(sf, csu0)) == 0
-end
-
-# SimpleHalley
-function benchmark_scalar(f, u0)
-    probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, SimpleHalley()))
-end
+using BenchmarkTools, LinearSolve, NonlinearSolve, StaticArrays, Random, LinearAlgebra,
+    Test, ForwardDiff, DiffEqBase
+
+_nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x))
+
+quadratic_f(u, p) = u .* u .- p
+quadratic_f!(du, u, p) = (du .= u .* u .- p)
+quadratic_f2(u, p) = @. p[1] * u * u - p[2]
+
+function newton_fails(u, p)
+    return 0.010000000000000002 .+
+           10.000000000000002 ./ (1 .+
+            (0.21640425613334457 .+
+             216.40425613334457 ./ (1 .+
+              (0.21640425613334457 .+
+               216.40425613334457 ./
+               (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
+           0.0011552453009332421u .- p
+end
+
+const TERMINATION_CONDITIONS = [
+    NormTerminationMode(), RelTerminationMode(), RelNormTerminationMode(),
+    AbsTerminationMode(), AbsNormTerminationMode(), RelSafeTerminationMode(),
+    AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode(),
+]
 
-function ff(u, p)
-    u .* u .- 2
-end
-const cu0 = @SVector[1.0, 1.0]
-function sf(u, p)
-    u * u - 2
-end
-const csu0 = 1.0
+# --- SimpleNewtonRaphson tests ---
 
-sol = benchmark_scalar(sf, csu0)
-@test sol.retcode === ReturnCode.Success
-@test sol.u * sol.u - 2 < 1e-9
+@testset "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion)
+    # Eval else the alg is type unstable
+    @eval begin
+        function benchmark_nlsolve_oop(f, u0, p = 2.0; autodiff = AutoForwardDiff())
+            prob = NonlinearProblem{false}(f, u0, p)
+            return solve(prob, $(alg)(; autodiff), abstol = 1e-9)
+        end
 
-sol = benchmark_scalar(ff, cu0)
-@test sol.retcode === ReturnCode.Success
-@test sol.u .* sol.u .- 2 < [1e-9, 1e-9]
+        function benchmark_nlsolve_iip(f, u0, p = 2.0; autodiff = AutoForwardDiff())
+            prob = NonlinearProblem{true}(f, u0, p)
+            return solve(prob, $(alg)(; autodiff), abstol = 1e-9)
+        end
+    end
 
-if VERSION >= v"1.7"
-    @test (@ballocated benchmark_scalar(sf, csu0)) == 0
-end
+    @testset "AutoDiff: $(_nameof(autodiff))" for autodiff in (AutoFiniteDiff(),
+        AutoForwardDiff())
+        @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
+            sol = benchmark_nlsolve_oop(quadratic_f, u0; autodiff)
+            @test SciMLBase.successful_retcode(sol)
+            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+        end
 
-# Broyden
-function benchmark_scalar(f, u0, alg)
-    probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, alg))
-end
+        @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
+            sol = benchmark_nlsolve_iip(quadratic_f!, u0; autodiff)
+            @test SciMLBase.successful_retcode(sol)
+            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+        end
+    end
 
-for alg in BROYDEN_SOLVERS
-    sol = benchmark_scalar(sf, csu0, alg)
-    @test sol.retcode === ReturnCode.Success
-    @test sol.u * sol.u - 2 < 1e-9
-    # FIXME: Termination Condition Implementation is allocating. Not sure how to fix it.
-    # if VERSION >= v"1.7"
-    #     @test (@ballocated benchmark_scalar($sf, $csu0, $termination_condition)) == 0
-    # end
-end
+    @testset "Allocations: Static Array and Scalars" begin
+        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, $(@SVector[1.0, 1.0]),
+            2.0; autodiff = AutoForwardDiff())) < 200
+        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, 1.0, 2.0;
+            autodiff = AutoForwardDiff())) == 0
+    end
 
-# Klement
-function benchmark_scalar(f, u0)
-    probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, Klement()))
-end
+    @testset "[OOP] Immutable AD" begin
+        for p in [1.0, 100.0]
+            @test begin
+                res = benchmark_nlsolve_oop(quadratic_f, @SVector[1.0, 1.0], p)
+                res_true = sqrt(p)
+                all(res.u .≈ res_true)
+            end
+            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
+                @SVector[1.0, 1.0], p).u[end], p) ≈ 1 / (2 * sqrt(p))
+        end
+    end
 
-sol = benchmark_scalar(sf, csu0)
-@test sol.retcode === ReturnCode.Success
-@test sol.u * sol.u - 2 < 1e-9
-if VERSION >= v"1.7"
-    @test (@ballocated benchmark_scalar(sf, csu0)) == 0
-end
+    @testset "[OOP] Scalar AD" begin
+        for p in 1.0:0.1:100.0
+            @test begin
+                res = benchmark_nlsolve_oop(quadratic_f, 1.0, p)
+                res_true = sqrt(p)
+                res.u ≈ res_true
+            end
+            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f, 1.0, p).u,
+                p) ≈ 1 / (2 * sqrt(p))
+        end
+    end
 
-# SimpleTrustRegion
-function benchmark_scalar(f, u0)
-    probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, SimpleTrustRegion()))
-end
+    t = (p) -> [sqrt(p[2] / p[1])]
+    p = [0.9, 50.0]
+    @test benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u ≈ sqrt(p[2] / p[1])
+    @test ForwardDiff.jacobian(p -> [benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u],
+        p) ≈ ForwardDiff.jacobian(t, p)
 
-sol = benchmark_scalar(sf, csu0)
-@test sol.retcode === ReturnCode.Success
-@test sol.u * sol.u - 2 < 1e-9
+    @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
+        u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
-# SimpleDFSane
-function benchmark_scalar(f, u0)
-    probN = NonlinearProblem{false}(f, u0)
-    sol = (solve(probN, SimpleDFSane()))
+        probN = NonlinearProblem(quadratic_f, u0, 2.0)
+        @test all(solve(probN, alg(); termination_condition).u .≈ sqrt(2.0))
+    end
 end
 
-sol = benchmark_scalar(sf, csu0)
-@test sol.retcode === ReturnCode.Success
-@test sol.u * sol.u - 2 < 1e-9
-
-# AD Tests
-using ForwardDiff
-
-# Immutable
-f, u0 = (u, p) -> u .* u .- p, @SVector[1.0, 1.0]
+# --- SimpleHalley tests ---
 
-for alg in (SimpleNewtonRaphson(), LBroyden(), Klement(), SimpleTrustRegion(),
-    SimpleDFSane(), SimpleHalley(), BROYDEN_SOLVERS...)
-    g = function (p)
-        probN = NonlinearProblem{false}(f, csu0, p)
-        sol = solve(probN, alg, abstol = 1e-9)
-        return sol.u[end]
+@testset "SimpleHalley" begin
+    function benchmark_nlsolve_oop(f, u0, p = 2.0; autodiff = AutoForwardDiff())
+        prob = NonlinearProblem{false}(f, u0, p)
+        return solve(prob, SimpleHalley(; autodiff), abstol = 1e-9)
     end
 
-    for p in 1.1:0.1:100.0
-        res = abs.(g(p))
-        # Not surprising if LBrouden fails to converge
-        if any(x -> isnan(x) || x <= 1e-5 || x >= 1e5, res) && alg isa LBroyden
-            @test_broken res ≈ sqrt(p)
-            @test_broken abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
-        else
-            @test res ≈ sqrt(p)
-            @test abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
+    @testset "AutoDiff: $(_nameof(autodiff))" for autodiff in (AutoFiniteDiff(),
+        AutoForwardDiff())
+        @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
+            sol = benchmark_nlsolve_oop(quadratic_f, u0; autodiff)
+            @test SciMLBase.successful_retcode(sol)
+            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
         end
     end
-end
 
-# Scalar
-f, u0 = (u, p) -> u * u - p, 1.0
-for alg in (SimpleNewtonRaphson(), Klement(), SimpleTrustRegion(),
-    SimpleDFSane(), SimpleHalley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
-    g = function (p)
-        probN = NonlinearProblem{false}(f, oftype(p, u0), p)
-        sol = solve(probN, alg)
-        return sol.u
+    @testset "Allocations: Static Array and Scalars" begin
+        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, 1.0, 2.0;
+            autodiff = AutoForwardDiff())) == 0
     end
 
-    for p in 1.1:0.1:100.0
-        res = abs.(g(p))
-        # Not surprising if LBrouden fails to converge
-        if any(x -> isnan(x) || x <= 1e-5 || x >= 1e5, res) && alg isa LBroyden
-            @test_broken res ≈ sqrt(p)
-            @test_broken abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
-        else
-            @test res ≈ sqrt(p)
-            @test abs.(ForwardDiff.derivative(g, p)) ≈ 1 / (2 * sqrt(p))
+    @testset "[OOP] Immutable AD" begin
+        for p in [1.0, 100.0]
+            @test begin
+                res = benchmark_nlsolve_oop(quadratic_f, @SVector[1.0, 1.0], p)
+                res_true = sqrt(p)
+                all(res.u .≈ res_true)
+            end
+            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
+                @SVector[1.0, 1.0], p).u[end], p) ≈ 1 / (2 * sqrt(p))
         end
     end
-end
-
-tspan = (1.0, 20.0)
-# Falsi
-g = function (p)
-    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-    sol = solve(probN, Falsi())
-    return sol.left
-end
-
-for p in 1.1:0.1:100.0
-    @test g(p) ≈ sqrt(p)
-    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-end
-
-# Ridder
-g = function (p)
-    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-    sol = solve(probN, Ridder())
-    return sol.left
-end
-
-for p in 1.1:0.1:100.0
-    @test g(p) ≈ sqrt(p)
-    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-end
-
-# Brent
-g = function (p)
-    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-    sol = solve(probN, Brent())
-    return sol.left
-end
-
-for p in 1.1:0.1:100.0
-    @test g(p) ≈ sqrt(p)
-    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-end
 
-# ITP
-g = function (p)
-    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-    sol = solve(probN, ITP())
-    return sol.u
-end
+    @testset "[OOP] Scalar AD" begin
+        for p in 1.0:0.1:100.0
+            @test begin
+                res = benchmark_nlsolve_oop(quadratic_f, 1.0, p)
+                res_true = sqrt(p)
+                res.u ≈ res_true
+            end
+            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f, 1.0, p).u,
+                p) ≈ 1 / (2 * sqrt(p))
+        end
+    end
 
-for p in 1.1:0.1:100.0
-    @test g(p) ≈ sqrt(p)
-    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-end
+    t = (p) -> [sqrt(p[2] / p[1])]
+    p = [0.9, 50.0]
+    @test benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u ≈ sqrt(p[2] / p[1])
+    @test ForwardDiff.jacobian(p -> [benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u],
+        p) ≈ ForwardDiff.jacobian(t, p)
 
-# Alefeld
-g = function (p)
-    probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-    sol = solve(probN, Alefeld())
-    return sol.u
-end
+    @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
+        u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
-for p in 1.1:0.1:100.0
-    @test g(p) ≈ sqrt(p)
-    @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+        probN = NonlinearProblem(quadratic_f, u0, 2.0)
+        @show solve(probN, SimpleHalley(); termination_condition).u
+        @test all(solve(probN, SimpleHalley(); termination_condition).u .≈ sqrt(2.0))
+    end
 end
 
-f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
-t = (p) -> [sqrt(p[2] / p[1])]
-p = [0.9, 50.0]
-g = function (p)
-    probN = IntervalNonlinearProblem{false}(f, tspan, p)
-    sol = solve(probN, Alefeld())
-    return [sol.u]
-end
+# --- SimpleBroyden / SimpleKlement / SimpleLimitedMemoryBroyden tests ---
 
-@test g(p) ≈ [sqrt(p[2] / p[1])]
-@test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
-
-f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
-t = (p) -> [sqrt(p[2] / p[1])]
-p = [0.9, 50.0]
-for alg in [Bisection(), Falsi(), Ridder(), Brent(), ITP()]
-    global g, p
-    g = function (p)
-        probN = IntervalNonlinearProblem{false}(f, tspan, p)
-        sol = solve(probN, alg)
-        return [sol.left]
+@testset "$(alg)" for alg in [SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
+        SimpleLimitedMemoryBroyden()]
+    function benchmark_nlsolve_oop(f, u0, p = 2.0)
+        prob = NonlinearProblem{false}(f, u0, p)
+        return solve(prob, alg, abstol = 1e-9)
     end
 
-    @test g(p) ≈ [sqrt(p[2] / p[1])]
-    @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
-end
-
-for alg in (SimpleNewtonRaphson(), Klement(), SimpleTrustRegion(),
-    SimpleDFSane(), SimpleHalley(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
-    global g, p
-    g = function (p)
-        probN = NonlinearProblem{false}(f, 0.5, p)
-        sol = solve(probN, alg)
-        return [abs(sol.u)]
+    function benchmark_nlsolve_iip(f, u0, p = 2.0)
+        prob = NonlinearProblem{true}(f, u0, p)
+        return solve(prob, alg, abstol = 1e-9)
     end
-    @test g(p) ≈ [sqrt(p[2] / p[1])]
-    @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
-end
-
-# Error Checks
-f, u0 = (u, p) -> u .* u .- 2.0, @SVector[1.0, 1.0]
-probN = NonlinearProblem(f, u0)
-
-for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
-    SimpleTrustRegion(),
-    SimpleTrustRegion(; autodiff = false), SimpleHalley(), SimpleHalley(; autodiff = false),
-    Klement(), SimpleDFSane(),
-    BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
-    sol = solve(probN, alg)
-
-    @test sol.retcode == ReturnCode.Success
-    @test sol.u[end] ≈ sqrt(2.0)
-end
-
-for u0 in [1.0, [1, 1.0]]
-    local f, probN, sol
-    f = (u, p) -> u .* u .- 2.0
-    probN = NonlinearProblem(f, u0)
-    sol = sqrt(2) * u0
 
-    for alg in (SimpleNewtonRaphson(), SimpleNewtonRaphson(; autodiff = false),
-        SimpleTrustRegion(), SimpleTrustRegion(; autodiff = false), Klement(),
-        SimpleDFSane(), BROYDEN_SOLVERS..., LBROYDEN_SOLVERS...)
-        sol2 = solve(probN, alg)
-
-        @test sol2.retcode == ReturnCode.Success
-        @test sol2.u ≈ sol
+    @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
+        sol = benchmark_nlsolve_oop(quadratic_f, u0)
+        @test SciMLBase.successful_retcode(sol)
+        @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
     end
-end
-
-# Bisection Tests
-f, tspan = (u, p) -> u .* u .- 2.0, (1.0, 2.0)
-probB = IntervalNonlinearProblem(f, tspan)
-
-# Falsi
-sol = solve(probB, Falsi())
-@test sol.left ≈ sqrt(2.0)
-
-# Bisection
-sol = solve(probB, Bisection())
-@test sol.left ≈ sqrt(2.0)
-
-# Ridder
-sol = solve(probB, Ridder())
-@test sol.left ≈ sqrt(2.0)
-tspan = (sqrt(2.0), 10.0)
-probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, Ridder())
-@test sol.left ≈ sqrt(2.0)
-tspan = (0.0, sqrt(2.0))
-probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, Ridder())
-@test sol.left ≈ sqrt(2.0)
-
-# Brent
-sol = solve(probB, Brent())
-@test sol.left ≈ sqrt(2.0)
-tspan = (sqrt(2.0), 10.0)
-probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, Brent())
-@test sol.left ≈ sqrt(2.0)
-tspan = (0.0, sqrt(2.0))
-probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, Brent())
-@test sol.left ≈ sqrt(2.0)
-
-# Alefeld
-sol = solve(probB, Alefeld())
-@test sol.u ≈ sqrt(2.0)
-tspan = (sqrt(2.0), 10.0)
-probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, Alefeld())
-@test sol.u ≈ sqrt(2.0)
-tspan = (0.0, sqrt(2.0))
-probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, Alefeld())
-@test sol.u ≈ sqrt(2.0)
-
-# ITP
-sol = solve(probB, ITP())
-@test sol.u ≈ sqrt(2.0)
-tspan = (sqrt(2.0), 10.0)
-probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, ITP())
-@test sol.u ≈ sqrt(2.0)
-tspan = (0.0, sqrt(2.0))
-probB = IntervalNonlinearProblem(f, tspan)
-sol = solve(probB, ITP())
-@test sol.u ≈ sqrt(2.0)
-
-# Tolerance tests for Interval methods
-f, tspan = (u, p) -> u .* u .- 2.0, (1.0, 10.0)
-probB = IntervalNonlinearProblem(f, tspan)
-tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
-ϵ = eps(1.0) #least possible tol for all methods
-
-for atol in tols
-    sol = solve(probB, Bisection(), abstol = atol)
-    @test abs(sol.u - sqrt(2)) < atol
-    @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
-    sol = solve(probB, Falsi(), abstol = atol)
-    @test abs(sol.u - sqrt(2)) < atol
-    @test abs(sol.u - sqrt(2)) > ϵ
-    sol = solve(probB, ITP(), abstol = atol)
-    @test abs(sol.u - sqrt(2)) < atol
-    @test abs(sol.u - sqrt(2)) > ϵ
-end
-
-tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution
-for atol in tols
-    sol = solve(probB, Ridder(), abstol = atol)
-    @test abs(sol.u - sqrt(2)) < atol
-    @test abs(sol.u - sqrt(2)) > ϵ
-    sol = solve(probB, Brent(), abstol = atol)
-    @test abs(sol.u - sqrt(2)) < atol
-    @test abs(sol.u - sqrt(2)) > ϵ
-end
 
-# Garuntee Tests for Bisection
-f = function (u, p)
-    if u < 2.0
-        return u - 2.0
-    elseif u > 3.0
-        return u - 3.0
-    else
-        return 0.0
+    @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
+        sol = benchmark_nlsolve_iip(quadratic_f!, u0)
+        @test SciMLBase.successful_retcode(sol)
+        @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
     end
-end
-probB = IntervalNonlinearProblem(f, (0.0, 4.0))
-
-sol = solve(probB, Bisection(; exact_left = true))
-@test f(sol.left, nothing) < 0.0
-@test f(nextfloat(sol.left), nothing) >= 0.0
-
-sol = solve(probB, Bisection(; exact_right = true))
-@test f(sol.right, nothing) >= 0.0
-@test f(prevfloat(sol.right), nothing) <= 0.0
-
-sol = solve(probB, Bisection(; exact_left = true, exact_right = true); immutable = false)
-@test f(sol.left, nothing) < 0.0
-@test f(nextfloat(sol.left), nothing) >= 0.0
-@test f(sol.right, nothing) >= 0.0
-@test f(prevfloat(sol.right), nothing) <= 0.0
-
-# Test that `SimpleTrustRegion` passes a test that `SimpleNewtonRaphson` fails on.
-u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
-global g, f
-f = (u, p) -> 0.010000000000000002 .+
-              10.000000000000002 ./ (1 .+
-               (0.21640425613334457 .+
-                216.40425613334457 ./ (1 .+
-                 (0.21640425613334457 .+
-                  216.40425613334457 ./
-                  (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
-              0.0011552453009332421u .- p
-g = function (p)
-    probN = NonlinearProblem{false}(f, u0, p)
-    sol = solve(probN, SimpleTrustRegion())
-    return sol.u
-end
-p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
-u = g(p)
-f(u, p)
-@test all(abs.(f(u, p)) .< 1e-10)
-
-# Test kwars in `SimpleTrustRegion`
-max_trust_radius = [10.0, 100.0, 1000.0]
-initial_trust_radius = [10.0, 1.0, 0.1]
-step_threshold = [0.0, 0.01, 0.25]
-shrink_threshold = [0.25, 0.3, 0.5]
-expand_threshold = [0.5, 0.8, 0.9]
-shrink_factor = [0.1, 0.3, 0.5]
-expand_factor = [1.5, 2.0, 3.0]
-max_shrink_times = [10, 20, 30]
-
-list_of_options = zip(max_trust_radius, initial_trust_radius, step_threshold,
-    shrink_threshold, expand_threshold, shrink_factor,
-    expand_factor, max_shrink_times)
-for options in list_of_options
-    local probN, sol, alg
-    alg = SimpleTrustRegion(max_trust_radius = options[1],
-        initial_trust_radius = options[2],
-        step_threshold = options[3],
-        shrink_threshold = options[4],
-        expand_threshold = options[5],
-        shrink_factor = options[6],
-        expand_factor = options[7],
-        max_shrink_times = options[8])
-
-    probN = NonlinearProblem(f, u0, p)
-    sol = solve(probN, alg)
-    @test all(abs.(f(u, p)) .< 1e-10)
-end
-
-# Test that `SimpleDFSane` passes a test that `SimpleNewtonRaphson` fails on.
-u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
-global g, f
-f = (u, p) -> 0.010000000000000002 .+
-              10.000000000000002 ./ (1 .+
-               (0.21640425613334457 .+
-                216.40425613334457 ./ (1 .+
-                 (0.21640425613334457 .+
-                  216.40425613334457 ./
-                  (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
-              0.0011552453009332421u .- p
-g = function (p)
-    probN = NonlinearProblem{false}(f, u0, p)
-    sol = solve(probN, SimpleDFSane())
-    return sol.u
-end
-p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
-u = g(p)
-f(u, p)
-@test all(abs.(f(u, p)) .< 1e-10)
-
-# Test kwars in `SimpleDFSane`
-σ_min = [1e-10, 1e-5, 1e-4]
-σ_max = [1e10, 1e5, 1e4]
-σ_1 = [1.0, 0.5, 2.0]
-M = [10, 1, 100]
-γ = [1e-4, 1e-3, 1e-5]
-τ_min = [0.1, 0.2, 0.3]
-τ_max = [0.5, 0.8, 0.9]
-nexp = [2, 1, 2]
-η_strategy = [
-    (f_1, k, x, F) -> f_1 / k^2,
-    (f_1, k, x, F) -> f_1 / k^3,
-    (f_1, k, x, F) -> f_1 / k^4,
-]
 
-list_of_options = zip(σ_min, σ_max, σ_1, M, γ, τ_min, τ_max, nexp,
-    η_strategy)
-for options in list_of_options
-    local probN, sol, alg
-    alg = SimpleDFSane(σ_min = options[1],
-        σ_max = options[2],
-        σ_1 = options[3],
-        M = options[4],
-        γ = options[5],
-        τ_min = options[6],
-        τ_max = options[7],
-        nexp = options[8],
-        η_strategy = options[9])
-
-    probN = NonlinearProblem(f, u0, p)
-    sol = solve(probN, alg)
-    @test all(abs.(f(u, p)) .< 1e-10)
-end
-
-f, u0 = (u, p) -> u .* u .- p, randn(1, 3)
-
-p = [2.0 1.0 5.0];
-probN = NonlinearProblem{false}(f, u0, p);
-
-sol = solve(probN, Broyden(batched = true))
-
-@test abs.(sol.u) ≈ sqrt.(p)
+    @testset "Allocations: Static Array and Scalars" begin
+        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, $(@SVector[1.0, 1.0]),
+            2.0)) < 200
+        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, 1.0, 2.0)) == 0
+    end
 
-@testset "Batched Solver: $(nameof(typeof(alg)))" for alg in (BATCHED_BROYDEN_SOLVERS...,
-    BATCHED_LBROYDEN_SOLVERS...,
-    BATCHED_DFSANE_SOLVERS...,
-    BATCHED_RAPHSON_SOLVERS...)
-    sol = solve(probN, alg; abstol = 1e-3, reltol = 1e-3)
+    @testset "[OOP] Immutable AD" begin
+        for p in [1.0, 100.0]
+            @test begin
+                res = benchmark_nlsolve_oop(quadratic_f, @SVector[1.0, 1.0], p)
+                res_true = sqrt(p)
+                all(res.u .≈ res_true)
+            end
+            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
+                @SVector[1.0, 1.0], p).u[end], p) ≈ 1 / (2 * sqrt(p))
+        end
+    end
 
-    @test sol.retcode == ReturnCode.Success
-    @test abs.(sol.u)≈sqrt.(p) atol=1e-3 rtol=1e-3
-end
+    @testset "[OOP] Scalar AD" begin
+        for p in 1.0:0.1:100.0
+            @test begin
+                res = benchmark_nlsolve_oop(quadratic_f, 1.0, p)
+                res_true = sqrt(p)
+                res.u ≈ res_true
+            end
+            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f, 1.0, p).u,
+                p) ≈ 1 / (2 * sqrt(p))
+        end
+    end
 
-## User specified Jacobian
+    t = (p) -> [sqrt(p[2] / p[1])]
+    p = [0.9, 50.0]
+    @test benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u ≈ sqrt(p[2] / p[1])
+    @test ForwardDiff.jacobian(p -> [benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u],
+        p) ≈ ForwardDiff.jacobian(t, p)
 
-f, u0 = (u, p) -> u .* u .- p, randn(3)
+    @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
+        u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
-f_jac(u, p) = begin
-    diagm(2 * u)
+        probN = NonlinearProblem(quadratic_f, u0, 2.0)
+        @test all(solve(probN, alg; termination_condition).u .≈ sqrt(2.0))
+    end
 end
 
-p = [2.0, 1.0, 5.0];
 
-probN = NonlinearProblem(NonlinearFunction(f, jac = f_jac), u0, p)
-
-for alg in (SimpleNewtonRaphson(), SimpleTrustRegion())
-    sol = solve(probN, alg)
-    @test abs.(sol.u) ≈ sqrt.(p)
-end
-
-# Flipped signs & reversed tspan test for bracketing algorithms
-f1(u, p) = u * u - p
-f2(u, p) = p - u * u
-
-for alg in (Alefeld(), Bisection(), Falsi(), Brent(), ITP(), Ridder())
-    for p in 1:4
-        inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
-        inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
-        inp3 = IntervalNonlinearProblem(f1, (2.0, 1.0), p)
-        inp4 = IntervalNonlinearProblem(f2, (2.0, 1.0), p)
-        @test abs.(solve(inp1, alg).u) ≈ sqrt.(p)
-        @test abs.(solve(inp2, alg).u) ≈ sqrt.(p)
-        @test abs.(solve(inp3, alg).u) ≈ sqrt.(p)
-        @test abs.(solve(inp4, alg).u) ≈ sqrt.(p)
-    end
-end
+1 + 1 + 1
+
+# tspan = (1.0, 20.0)
+# # Falsi
+# g = function (p)
+#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
+#     sol = solve(probN, Falsi())
+#     return sol.left
+# end
+
+# for p in 1.1:0.1:100.0
+#     @test g(p) ≈ sqrt(p)
+#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+# end
+
+# # Ridder
+# g = function (p)
+#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
+#     sol = solve(probN, Ridder())
+#     return sol.left
+# end
+
+# for p in 1.1:0.1:100.0
+#     @test g(p) ≈ sqrt(p)
+#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+# end
+
+# # Brent
+# g = function (p)
+#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
+#     sol = solve(probN, Brent())
+#     return sol.left
+# end
+
+# for p in 1.1:0.1:100.0
+#     @test g(p) ≈ sqrt(p)
+#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+# end
+
+# # ITP
+# g = function (p)
+#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
+#     sol = solve(probN, ITP())
+#     return sol.u
+# end
+
+# for p in 1.1:0.1:100.0
+#     @test g(p) ≈ sqrt(p)
+#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+# end
+
+# # Alefeld
+# g = function (p)
+#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
+#     sol = solve(probN, Alefeld())
+#     return sol.u
+# end
+
+# for p in 1.1:0.1:100.0
+#     @test g(p) ≈ sqrt(p)
+#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
+# end
+
+# f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
+# t = (p) -> [sqrt(p[2] / p[1])]
+# p = [0.9, 50.0]
+# g = function (p)
+#     probN = IntervalNonlinearProblem{false}(f, tspan, p)
+#     sol = solve(probN, Alefeld())
+#     return [sol.u]
+# end
+
+# @test g(p) ≈ [sqrt(p[2] / p[1])]
+# @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
+
+# f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
+# t = (p) -> [sqrt(p[2] / p[1])]
+# p = [0.9, 50.0]
+# for alg in [Bisection(), Falsi(), Ridder(), Brent(), ITP()]
+#     global g, p
+#     g = function (p)
+#         probN = IntervalNonlinearProblem{false}(f, tspan, p)
+#         sol = solve(probN, alg)
+#         return [sol.left]
+#     end
+
+#     @test g(p) ≈ [sqrt(p[2] / p[1])]
+#     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
+# end
+
+# # Bisection Tests
+# f, tspan = (u, p) -> u .* u .- 2.0, (1.0, 2.0)
+# probB = IntervalNonlinearProblem(f, tspan)
+
+# # Falsi
+# sol = solve(probB, Falsi())
+# @test sol.left ≈ sqrt(2.0)
+
+# # Bisection
+# sol = solve(probB, Bisection())
+# @test sol.left ≈ sqrt(2.0)
+
+# # Ridder
+# sol = solve(probB, Ridder())
+# @test sol.left ≈ sqrt(2.0)
+# tspan = (sqrt(2.0), 10.0)
+# probB = IntervalNonlinearProblem(f, tspan)
+# sol = solve(probB, Ridder())
+# @test sol.left ≈ sqrt(2.0)
+# tspan = (0.0, sqrt(2.0))
+# probB = IntervalNonlinearProblem(f, tspan)
+# sol = solve(probB, Ridder())
+# @test sol.left ≈ sqrt(2.0)
+
+# # Brent
+# sol = solve(probB, Brent())
+# @test sol.left ≈ sqrt(2.0)
+# tspan = (sqrt(2.0), 10.0)
+# probB = IntervalNonlinearProblem(f, tspan)
+# sol = solve(probB, Brent())
+# @test sol.left ≈ sqrt(2.0)
+# tspan = (0.0, sqrt(2.0))
+# probB = IntervalNonlinearProblem(f, tspan)
+# sol = solve(probB, Brent())
+# @test sol.left ≈ sqrt(2.0)
+
+# # Alefeld
+# sol = solve(probB, Alefeld())
+# @test sol.u ≈ sqrt(2.0)
+# tspan = (sqrt(2.0), 10.0)
+# probB = IntervalNonlinearProblem(f, tspan)
+# sol = solve(probB, Alefeld())
+# @test sol.u ≈ sqrt(2.0)
+# tspan = (0.0, sqrt(2.0))
+# probB = IntervalNonlinearProblem(f, tspan)
+# sol = solve(probB, Alefeld())
+# @test sol.u ≈ sqrt(2.0)
+
+# # ITP
+# sol = solve(probB, ITP())
+# @test sol.u ≈ sqrt(2.0)
+# tspan = (sqrt(2.0), 10.0)
+# probB = IntervalNonlinearProblem(f, tspan)
+# sol = solve(probB, ITP())
+# @test sol.u ≈ sqrt(2.0)
+# tspan = (0.0, sqrt(2.0))
+# probB = IntervalNonlinearProblem(f, tspan)
+# sol = solve(probB, ITP())
+# @test sol.u ≈ sqrt(2.0)
+
+# # Tolerance tests for Interval methods
+# f, tspan = (u, p) -> u .* u .- 2.0, (1.0, 10.0)
+# probB = IntervalNonlinearProblem(f, tspan)
+# tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
+# ϵ = eps(1.0) #least possible tol for all methods
+
+# for atol in tols
+#     sol = solve(probB, Bisection(), abstol = atol)
+#     @test abs(sol.u - sqrt(2)) < atol
+#     @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
+#     sol = solve(probB, Falsi(), abstol = atol)
+#     @test abs(sol.u - sqrt(2)) < atol
+#     @test abs(sol.u - sqrt(2)) > ϵ
+#     sol = solve(probB, ITP(), abstol = atol)
+#     @test abs(sol.u - sqrt(2)) < atol
+#     @test abs(sol.u - sqrt(2)) > ϵ
+# end
+
+# tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution
+# for atol in tols
+#     sol = solve(probB, Ridder(), abstol = atol)
+#     @test abs(sol.u - sqrt(2)) < atol
+#     @test abs(sol.u - sqrt(2)) > ϵ
+#     sol = solve(probB, Brent(), abstol = atol)
+#     @test abs(sol.u - sqrt(2)) < atol
+#     @test abs(sol.u - sqrt(2)) > ϵ
+# end
+
+# # Garuntee Tests for Bisection
+# f = function (u, p)
+#     if u < 2.0
+#         return u - 2.0
+#     elseif u > 3.0
+#         return u - 3.0
+#     else
+#         return 0.0
+#     end
+# end
+# probB = IntervalNonlinearProblem(f, (0.0, 4.0))
+
+# sol = solve(probB, Bisection(; exact_left = true))
+# @test f(sol.left, nothing) < 0.0
+# @test f(nextfloat(sol.left), nothing) >= 0.0
+
+# sol = solve(probB, Bisection(; exact_right = true))
+# @test f(sol.right, nothing) >= 0.0
+# @test f(prevfloat(sol.right), nothing) <= 0.0
+
+# sol = solve(probB, Bisection(; exact_left = true, exact_right = true); immutable = false)
+# @test f(sol.left, nothing) < 0.0
+# @test f(nextfloat(sol.left), nothing) >= 0.0
+# @test f(sol.right, nothing) >= 0.0
+# @test f(prevfloat(sol.right), nothing) <= 0.0
+
+# # Flipped signs & reversed tspan test for bracketing algorithms
+# f1(u, p) = u * u - p
+# f2(u, p) = p - u * u
+
+# for alg in (Alefeld(), Bisection(), Falsi(), Brent(), ITP(), Ridder())
+#     for p in 1:4
+#         inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
+#         inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
+#         inp3 = IntervalNonlinearProblem(f1, (2.0, 1.0), p)
+#         inp4 = IntervalNonlinearProblem(f2, (2.0, 1.0), p)
+#         @test abs.(solve(inp1, alg).u) ≈ sqrt.(p)
+#         @test abs.(solve(inp2, alg).u) ≈ sqrt.(p)
+#         @test abs.(solve(inp3, alg).u) ≈ sqrt.(p)
+#         @test abs.(solve(inp4, alg).u) ≈ sqrt.(p)
+#     end
+# end
diff --git a/lib/SimpleNonlinearSolve/test/inplace.jl b/lib/SimpleNonlinearSolve/test/inplace.jl
deleted file mode 100644
index 2e9d033a8..000000000
--- a/lib/SimpleNonlinearSolve/test/inplace.jl
+++ /dev/null
@@ -1,52 +0,0 @@
-using SimpleNonlinearSolve,
-    StaticArrays, BenchmarkTools, DiffEqBase, LinearAlgebra, Test,
-    NNlib
-
-# Supported Solvers: BatchedBroyden, BatchedSimpleDFSane, BatchedSimpleNewtonRaphson
-function f!(du::AbstractArray{<:Number, N},
-        u::AbstractArray{<:Number, N},
-        p::AbstractVector) where {N}
-    u_ = reshape(u, :, size(u, N))
-    du .= reshape(sum(abs2, u_; dims = 1) .- u_ .- reshape(p, 1, :), size(u))
-    return du
-end
-
-function f!(du::AbstractMatrix, u::AbstractMatrix, p::AbstractVector)
-    du .= sum(abs2, u; dims = 1) .- u .- reshape(p, 1, :)
-    return du
-end
-
-function f!(du::AbstractVector, u::AbstractVector, p::AbstractVector)
-    du .= sum(abs2, u) .- u .- p
-    return du
-end
-
-@testset "Solver: $(nameof(typeof(solver)))" for solver in (Broyden(; batched = true),
-    SimpleDFSane(; batched = true),
-    SimpleNewtonRaphson(; batched = true))
-    @testset "T: $T" for T in (Float32, Float64)
-        p = rand(T, 5)
-        @testset "size(u0): $sz" for sz in ((2, 5), (1, 5), (2, 3, 5))
-            u0 = ones(T, sz)
-            prob = NonlinearProblem{true}(f!, u0, p)
-
-            sol = solve(prob, solver)
-
-            @test SciMLBase.successful_retcode(sol.retcode)
-
-            @test sol.resid≈zero(sol.resid) atol=5e-3
-        end
-
-        p = rand(T, 1)
-        @testset "size(u0): $sz" for sz in ((3,), (5,), (10,))
-            u0 = ones(T, sz)
-            prob = NonlinearProblem{true}(f!, u0, p)
-
-            sol = solve(prob, solver)
-
-            @test SciMLBase.successful_retcode(sol.retcode)
-
-            @test sol.resid≈zero(sol.resid) atol=5e-3
-        end
-    end
-end
diff --git a/lib/SimpleNonlinearSolve/test/least_squares.jl b/lib/SimpleNonlinearSolve/test/least_squares.jl
index a7003f697..e09ad92bc 100644
--- a/lib/SimpleNonlinearSolve/test/least_squares.jl
+++ b/lib/SimpleNonlinearSolve/test/least_squares.jl
@@ -13,7 +13,9 @@ end
 
 θ_init = θ_true .+ 0.1
 prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
-sol = solve(prob_oop, SimpleNewtonRaphson())
-sol = solve(prob_oop, SimpleGaussNewton())
 
-@test norm(sol.resid) < 1e-12
+for solver in [SimpleNewtonRaphson(AutoForwardDiff()), SimpleGaussNewton(AutoForwardDiff()),
+        SimpleNewtonRaphson(AutoFiniteDiff()), SimpleGaussNewton(AutoFiniteDiff())]
+    sol = solve(prob_oop, solver)
+    @test norm(sol.resid) < 1e-12
+end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index d0fd1ff9b..a38e954d8 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -5,7 +5,6 @@ const GROUP = get(ENV, "GROUP", "All")
 @time begin
     if GROUP == "All" || GROUP == "Core"
         @time @safetestset "Basic Tests + Some AD" include("basictests.jl")
-        @time @safetestset "Inplace Tests" include("inplace.jl")
         @time @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
         @time @safetestset "Least Squares Tests" include("least_squares.jl")
     end

From c1d3e8c14f2f5ae0def54d84499b8b6f4597046b Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 Nov 2023 01:55:44 -0500
Subject: [PATCH 235/375] Fix the tests

---
 lib/SimpleNonlinearSolve/src/ad.jl            |   3 +-
 .../src/bracketing/bisection.jl               |  10 +-
 .../src/nlsolve/broyden.jl                    |   4 +-
 .../src/nlsolve/dfsane.jl                     |   3 +
 .../src/nlsolve/halley.jl                     |   4 +-
 .../src/nlsolve/klement.jl                    |   4 +-
 .../src/nlsolve/trustRegion.jl                |   9 +-
 .../test/23_test_problems.jl                  |  87 +++++
 lib/SimpleNonlinearSolve/test/Project.toml    |   1 +
 lib/SimpleNonlinearSolve/test/basictests.jl   | 308 +++++++-----------
 .../test/matrix_resizing_tests.jl             |   3 +-
 lib/SimpleNonlinearSolve/test/runtests.jl     |   5 +-
 12 files changed, 228 insertions(+), 213 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/test/23_test_problems.jl

diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 8cbff710c..d4cbcf744 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -1,9 +1,8 @@
 function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     f = prob.f
     p = value(prob.p)
-    u0 = value(prob.u0)
     if prob isa IntervalNonlinearProblem
-        tspan = value(prob.tspan)
+        tspan = value.(prob.tspan)
         newprob = IntervalNonlinearProblem(f, tspan, p; prob.kwargs...)
     else
         u0 = value(prob.u0)
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
index 42bb2cad0..66418b3e0 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
@@ -86,16 +86,18 @@ function __bisection(left, right, fl, fr, f::F; abstol, maxiters, prob, alg) whe
         end
 
         fm = f(mid)
-        if abs((right - left) / 2) < abstol || abs(fm) < abstol
+        if abs((right - left) / 2) < abstol
             sol = build_solution(prob, alg, mid, fm; left, right,
                 retcode = ReturnCode.Success)
             break
         end
 
-        if sign(fl * fm) < 0
-            right, fr = mid, fm
+        if iszero(fm)
+            right = mid
+            fr = fm
         else
-            left, fl = mid, fm
+            left = mid
+            fl = fm
         end
 
         i += 1
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index aaf959cb5..1e544a4d2 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -43,7 +43,9 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
 
         @bb @. δJ⁻¹n = (δx - J⁻¹δf) / d
 
-        @bb δJ⁻¹ = δJ⁻¹n × transpose(xᵀJ⁻¹)
+        δJ⁻¹n_ = _vec(δJ⁻¹n)
+        xᵀJ⁻¹_ = _vec(xᵀJ⁻¹)
+        @bb δJ⁻¹ = δJ⁻¹n_ × transpose(xᵀJ⁻¹_)
         @bb J⁻¹ .+= δJ⁻¹
 
         @bb copyto!(xo, x)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 657f760cb..77ee497a3 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -72,6 +72,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
     fx_norm = norm(fx)^nexp
     α_1 = one(T)
     f_1 = fx_norm
+
     history_f_k = fill(fx_norm, M)
 
     # Generate the cache
@@ -118,6 +119,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
 
             fx = __eval_f(prob, fx, x)
             fx_norm_new = norm(fx)^nexp
+
+            k += 1
         end
 
         tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 3e6e4d55f..161abaed1 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -58,7 +58,9 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
         @bb A .*= -1
         bᵢ = dfx \ Aaᵢ
 
-        @bb @. cᵢ = (aᵢ * aᵢ) / (-aᵢ + (T(0.5) * bᵢ))
+        cᵢ_ = _vec(cᵢ)
+        @bb @. cᵢ_ = (aᵢ * aᵢ) / (-aᵢ + (T(0.5) * bᵢ))
+        cᵢ = _restructure(cᵢ, cᵢ_)
 
         if i == 1
             if iszero(fx)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index 56d6ccd55..5041dc4df 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -55,9 +55,9 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
 
         @bb copyto!(δx, fprev)
         if setindex_trait(δx) === CanSetindex()
-            ldiv!(F_, δx)
+            ldiv!(F_, _vec(δx))
         else
-            δx = F_ \ δx
+            δx = _restructure(δx, F_ \ _vec(δx))
         end
         @bb @. x = xo - δx
         fx = __eval_f(prob, fx, x)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 3c3ad60b4..bf85dcfe3 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -76,7 +76,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
 
     fₖ = 0.5 * norm(fx)^2
     H = ∇f' * ∇f
-    g = ∇f' * fx
+    g = _restructure(x, ∇f' * _vec(fx))
     shrink_counter = 0
 
     @bb δsd = copy(x)
@@ -96,7 +96,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
         fₖ₊₁ = norm(fx)^2 / T(2)
 
         # Compute the ratio of the actual to predicted reduction.
-        @bb Hδ = H × δ
+        # @show size(H), size(δ)
+        @bb Hδ = H × vec(δ)
         r = (fₖ₊₁ - fₖ) / (dot(δ', g) + dot(δ', Hδ) / T(2))
 
         # Update the trust region radius.
@@ -124,7 +125,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
             fₖ = fₖ₊₁
 
             @bb H = transpose(∇f) × ∇f
-            @bb g = transpose(∇f) × fx
+            @bb g = transpose(∇f) × vec(fx)
         end
     end
 
@@ -135,7 +136,7 @@ function dogleg_method!!(cache, J, f, g, Δ)
     (; δsd, δN_δsd, δN) = cache
 
     # Compute the Newton step.
-    @bb δN .= J \ f
+    @bb δN .= _restructure(δN, J \ _vec(f))
     @bb δN .*= -1
     # Test if the full step is within the trust region.
     (norm(δN) ≤ Δ) && return δN
diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
new file mode 100644
index 000000000..5edd5710b
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
@@ -0,0 +1,87 @@
+using SimpleNonlinearSolve, LinearAlgebra, NonlinearProblemLibrary, Test
+
+problems = NonlinearProblemLibrary.problems
+dicts = NonlinearProblemLibrary.dicts
+
+function test_on_library(problems, dicts, alg_ops, broken_tests, ϵ = 1e-4;
+        skip_tests = nothing)
+    for (idx, (problem, dict)) in enumerate(zip(problems, dicts))
+        x = dict["start"]
+        res = similar(x)
+        nlprob = NonlinearProblem(problem, copy(x))
+        @testset "$idx: $(dict["title"])" begin
+            for alg in alg_ops
+                try
+                    sol = solve(nlprob, alg;
+                        termination_condition = AbsNormTerminationMode())
+                    problem(res, sol.u, nothing)
+
+                    skip = skip_tests !== nothing && idx in skip_tests[alg]
+                    if skip
+                        @test_skip norm(res) ≤ ϵ
+                        continue
+                    end
+                    broken = idx in broken_tests[alg] ? true : false
+                    @test norm(res)≤ϵ broken=broken
+                catch
+                    broken = idx in broken_tests[alg] ? true : false
+                    if broken
+                        @test false broken=true
+                    else
+                        @test 1 == 2
+                    end
+                end
+            end
+        end
+    end
+end
+
+@testset "SimpleNewtonRaphson 23 Test Problems" begin
+    alg_ops = (SimpleNewtonRaphson(),)
+
+    # dictionary with indices of test problems where method does not converge to small residual
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [6]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testset "SimpleTrustRegion 23 Test Problems" begin
+    alg_ops = (SimpleTrustRegion(),)
+
+    # dictionary with indices of test problems where method does not converge to small residual
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [3, 6, 15, 16, 21]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testset "SimpleDFSane 23 Test Problems" begin
+    alg_ops = (SimpleDFSane(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 7, 9, 11, 12, 13, 15, 16, 17, 21, 22]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testset "SimpleBroyden 23 Test Problems" begin
+    alg_ops = (SimpleBroyden(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 6, 11, 12, 13, 14]
+
+    skip_tests = Dict(alg => Int[] for alg in alg_ops)
+    skip_tests[alg_ops[1]] = [22]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests; skip_tests)
+end
+
+@testset "SimpleKlement 23 Test Problems" begin
+    alg_ops = (SimpleKlement(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 13, 19, 21, 22]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
index 835a6aa43..b8072e68e 100644
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -4,6 +4,7 @@ DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
+NonlinearProblemLibrary = "b7050fa9-e91f-4b37-bcee-a89a063da141"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 4963a52fe..413a6a512 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,5 +1,5 @@
-using BenchmarkTools, LinearSolve, NonlinearSolve, StaticArrays, Random, LinearAlgebra,
-    Test, ForwardDiff, DiffEqBase
+using BenchmarkTools, LinearSolve, SimpleNonlinearSolve, StaticArrays, Random,
+    LinearAlgebra, Test, ForwardDiff, DiffEqBase
 
 _nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x))
 
@@ -163,8 +163,8 @@ end
 
 # --- SimpleBroyden / SimpleKlement / SimpleLimitedMemoryBroyden tests ---
 
-@testset "$(alg)" for alg in [SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
-        SimpleLimitedMemoryBroyden()]
+@testset "$(_nameof(alg))" for alg in [SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
+    SimpleLimitedMemoryBroyden()]
     function benchmark_nlsolve_oop(f, u0, p = 2.0)
         prob = NonlinearProblem{false}(f, u0, p)
         return solve(prob, alg, abstol = 1e-9)
@@ -190,30 +190,39 @@ end
     @testset "Allocations: Static Array and Scalars" begin
         @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, $(@SVector[1.0, 1.0]),
             2.0)) < 200
-        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, 1.0, 2.0)) == 0
+        allocs = alg isa SimpleDFSane ? 144 : 0
+        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, 1.0, 2.0)) == allocs
     end
 
     @testset "[OOP] Immutable AD" begin
         for p in [1.0, 100.0]
-            @test begin
-                res = benchmark_nlsolve_oop(quadratic_f, @SVector[1.0, 1.0], p)
-                res_true = sqrt(p)
-                all(res.u .≈ res_true)
+            res = benchmark_nlsolve_oop(quadratic_f, @SVector[1.0, 1.0], p)
+
+            if any(x -> isnan(x) || x <= 1e-5 || x >= 1e5, res)
+                @test_broken all(res .≈ sqrt(p))
+                @test_broken abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
+                    @SVector[1.0, 1.0], p).u[end], p)) ≈ 1 / (2 * sqrt(p))
+            else
+                @test all(res .≈ sqrt(p))
+                @test isapprox(abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
+                        @SVector[1.0, 1.0], p).u[end], p)), 1 / (2 * sqrt(p)))
             end
-            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
-                @SVector[1.0, 1.0], p).u[end], p) ≈ 1 / (2 * sqrt(p))
         end
     end
 
     @testset "[OOP] Scalar AD" begin
         for p in 1.0:0.1:100.0
-            @test begin
-                res = benchmark_nlsolve_oop(quadratic_f, 1.0, p)
-                res_true = sqrt(p)
-                res.u ≈ res_true
+            res = benchmark_nlsolve_oop(quadratic_f, 1.0, p)
+
+            if any(x -> isnan(x) || x <= 1e-5 || x >= 1e5, res)
+                @test_broken all(res .≈ sqrt(p))
+                @test_broken abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
+                        1.0, p).u, p)) ≈ 1 / (2 * sqrt(p))
+            else
+                @test all(res .≈ sqrt(p))
+                @test isapprox(abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
+                            1.0, p).u, p)), 1 / (2 * sqrt(p)))
             end
-            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f, 1.0, p).u,
-                p) ≈ 1 / (2 * sqrt(p))
         end
     end
 
@@ -231,185 +240,109 @@ end
     end
 end
 
+@testset "Newton Fails" begin
+    function benchmark_nlsolve_oop(f, u0, p, alg)
+        prob = NonlinearProblem{false}(f, u0, p)
+        return solve(prob, alg; abstol = 1e-9)
+    end
 
-1 + 1 + 1
+    u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
+    p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
 
-# tspan = (1.0, 20.0)
-# # Falsi
-# g = function (p)
-#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-#     sol = solve(probN, Falsi())
-#     return sol.left
-# end
+    for alg in (SimpleDFSane(), SimpleTrustRegion(), SimpleHalley())
+        sol = benchmark_nlsolve_oop(newton_fails, u0, p, alg)
+        @test SciMLBase.successful_retcode(sol)
+        @test all(abs.(newton_fails(sol.u, p)) .< 1e-9)
+    end
+end
 
-# for p in 1.1:0.1:100.0
-#     @test g(p) ≈ sqrt(p)
-#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-# end
+# --- Interval Nonlinear Problems ---
 
-# # Ridder
-# g = function (p)
-#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-#     sol = solve(probN, Ridder())
-#     return sol.left
-# end
+@testset "Interval Nonlinear Problem: $(alg)" for alg in (Bisection(), Falsi(), Ridder(),
+    Brent(), ITP(), Alefeld())
+    tspan = (1.0, 20.0)
 
-# for p in 1.1:0.1:100.0
-#     @test g(p) ≈ sqrt(p)
-#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-# end
+    function g(p)
+        probN = IntervalNonlinearProblem{false}(quadratic_f, typeof(p).(tspan), p)
+        sol = solve(probN, alg; abstol = 1e-9)
+        return sol.left
+    end
 
-# # Brent
-# g = function (p)
-#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-#     sol = solve(probN, Brent())
-#     return sol.left
-# end
+    for p in 1.1:0.1:100.0
+        @test g(p)≈sqrt(p) atol=1e-3 rtol=1e-3
+        @test ForwardDiff.derivative(g, p)≈1 / (2 * sqrt(p)) atol=1e-3 rtol=1e-3
+    end
 
-# for p in 1.1:0.1:100.0
-#     @test g(p) ≈ sqrt(p)
-#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-# end
+    t = (p) -> [sqrt(p[2] / p[1])]
+    p = [0.9, 50.0]
 
-# # ITP
-# g = function (p)
-#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-#     sol = solve(probN, ITP())
-#     return sol.u
-# end
+    function g2(p)
+        probN = IntervalNonlinearProblem{false}((u, p) -> p[1] * u * u - p[2], tspan, p)
+        sol = solve(probN, alg; abstol = 1e-9)
+        return [sol.u]
+    end
 
-# for p in 1.1:0.1:100.0
-#     @test g(p) ≈ sqrt(p)
-#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-# end
+    @test g2(p)≈[sqrt(p[2] / p[1])] atol=1e-3 rtol=1e-3
+    @test ForwardDiff.jacobian(g2, p)≈ForwardDiff.jacobian(t, p) atol=1e-3 rtol=1e-3
 
-# # Alefeld
-# g = function (p)
-#     probN = IntervalNonlinearProblem{false}(f, typeof(p).(tspan), p)
-#     sol = solve(probN, Alefeld())
-#     return sol.u
-# end
+    probB = IntervalNonlinearProblem{false}(quadratic_f, (1.0, 2.0), 2.0)
+    sol = solve(probB, alg; abstol = 1e-9)
+    @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
 
-# for p in 1.1:0.1:100.0
-#     @test g(p) ≈ sqrt(p)
-#     @test ForwardDiff.derivative(g, p) ≈ 1 / (2 * sqrt(p))
-# end
+    if !(alg isa Bisection || alg isa Falsi)
+        probB = IntervalNonlinearProblem{false}(quadratic_f, (sqrt(2.0), 10.0), 2.0)
+        sol = solve(probB, alg; abstol = 1e-9)
+        @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
 
-# f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
-# t = (p) -> [sqrt(p[2] / p[1])]
-# p = [0.9, 50.0]
-# g = function (p)
-#     probN = IntervalNonlinearProblem{false}(f, tspan, p)
-#     sol = solve(probN, Alefeld())
-#     return [sol.u]
-# end
+        probB = IntervalNonlinearProblem{false}(quadratic_f, (0.0, sqrt(2.0)), 2.0)
+        sol = solve(probB, alg; abstol = 1e-9)
+        @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
+    end
+end
 
-# @test g(p) ≈ [sqrt(p[2] / p[1])]
-# @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
-
-# f, tspan = (u, p) -> p[1] * u * u - p[2], (1.0, 100.0)
-# t = (p) -> [sqrt(p[2] / p[1])]
-# p = [0.9, 50.0]
-# for alg in [Bisection(), Falsi(), Ridder(), Brent(), ITP()]
-#     global g, p
-#     g = function (p)
-#         probN = IntervalNonlinearProblem{false}(f, tspan, p)
-#         sol = solve(probN, alg)
-#         return [sol.left]
-#     end
+@testset "Tolerance Tests Interval Methods: $(alg)" for alg in (Bisection(), Falsi(), ITP())
+    probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
+    tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
+    ϵ = eps(1.0) #least possible tol for all methods
 
-#     @test g(p) ≈ [sqrt(p[2] / p[1])]
-#     @test ForwardDiff.jacobian(g, p) ≈ ForwardDiff.jacobian(t, p)
-# end
+    for atol in tols
+        sol = solve(probB, alg; abstol = atol)
+        @test abs(sol.u - sqrt(2)) < atol
+        @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
+    end
+end
 
-# # Bisection Tests
-# f, tspan = (u, p) -> u .* u .- 2.0, (1.0, 2.0)
-# probB = IntervalNonlinearProblem(f, tspan)
-
-# # Falsi
-# sol = solve(probB, Falsi())
-# @test sol.left ≈ sqrt(2.0)
-
-# # Bisection
-# sol = solve(probB, Bisection())
-# @test sol.left ≈ sqrt(2.0)
-
-# # Ridder
-# sol = solve(probB, Ridder())
-# @test sol.left ≈ sqrt(2.0)
-# tspan = (sqrt(2.0), 10.0)
-# probB = IntervalNonlinearProblem(f, tspan)
-# sol = solve(probB, Ridder())
-# @test sol.left ≈ sqrt(2.0)
-# tspan = (0.0, sqrt(2.0))
-# probB = IntervalNonlinearProblem(f, tspan)
-# sol = solve(probB, Ridder())
-# @test sol.left ≈ sqrt(2.0)
-
-# # Brent
-# sol = solve(probB, Brent())
-# @test sol.left ≈ sqrt(2.0)
-# tspan = (sqrt(2.0), 10.0)
-# probB = IntervalNonlinearProblem(f, tspan)
-# sol = solve(probB, Brent())
-# @test sol.left ≈ sqrt(2.0)
-# tspan = (0.0, sqrt(2.0))
-# probB = IntervalNonlinearProblem(f, tspan)
-# sol = solve(probB, Brent())
-# @test sol.left ≈ sqrt(2.0)
-
-# # Alefeld
-# sol = solve(probB, Alefeld())
-# @test sol.u ≈ sqrt(2.0)
-# tspan = (sqrt(2.0), 10.0)
-# probB = IntervalNonlinearProblem(f, tspan)
-# sol = solve(probB, Alefeld())
-# @test sol.u ≈ sqrt(2.0)
-# tspan = (0.0, sqrt(2.0))
-# probB = IntervalNonlinearProblem(f, tspan)
-# sol = solve(probB, Alefeld())
-# @test sol.u ≈ sqrt(2.0)
-
-# # ITP
-# sol = solve(probB, ITP())
-# @test sol.u ≈ sqrt(2.0)
-# tspan = (sqrt(2.0), 10.0)
-# probB = IntervalNonlinearProblem(f, tspan)
-# sol = solve(probB, ITP())
-# @test sol.u ≈ sqrt(2.0)
-# tspan = (0.0, sqrt(2.0))
-# probB = IntervalNonlinearProblem(f, tspan)
-# sol = solve(probB, ITP())
-# @test sol.u ≈ sqrt(2.0)
-
-# # Tolerance tests for Interval methods
-# f, tspan = (u, p) -> u .* u .- 2.0, (1.0, 10.0)
-# probB = IntervalNonlinearProblem(f, tspan)
-# tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
-# ϵ = eps(1.0) #least possible tol for all methods
-
-# for atol in tols
-#     sol = solve(probB, Bisection(), abstol = atol)
-#     @test abs(sol.u - sqrt(2)) < atol
-#     @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
-#     sol = solve(probB, Falsi(), abstol = atol)
-#     @test abs(sol.u - sqrt(2)) < atol
-#     @test abs(sol.u - sqrt(2)) > ϵ
-#     sol = solve(probB, ITP(), abstol = atol)
-#     @test abs(sol.u - sqrt(2)) < atol
-#     @test abs(sol.u - sqrt(2)) > ϵ
-# end
+@testset "Tolerance Tests Interval Methods: $(alg)" for alg in (Ridder(), Brent())
+    probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
+    tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution
+    ϵ = eps(1.0) #least possible tol for all methods
 
-# tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution
-# for atol in tols
-#     sol = solve(probB, Ridder(), abstol = atol)
-#     @test abs(sol.u - sqrt(2)) < atol
-#     @test abs(sol.u - sqrt(2)) > ϵ
-#     sol = solve(probB, Brent(), abstol = atol)
-#     @test abs(sol.u - sqrt(2)) < atol
-#     @test abs(sol.u - sqrt(2)) > ϵ
-# end
+    for atol in tols
+        sol = solve(probB, alg; abstol = atol)
+        @test abs(sol.u - sqrt(2)) < atol
+        @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
+    end
+end
 
+@testset "Flipped Signs and Reversed Tspan: $(alg)" for alg in (Alefeld(), Bisection(),
+    Falsi(), Brent(), ITP(), Ridder())
+    f1(u, p) = u * u - p
+    f2(u, p) = p - u * u
+
+    for p in 1:4
+        inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
+        inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
+        inp3 = IntervalNonlinearProblem(f1, (2.0, 1.0), p)
+        inp4 = IntervalNonlinearProblem(f2, (2.0, 1.0), p)
+        @test abs.(solve(inp1, alg).u) ≈ sqrt.(p)
+        @test abs.(solve(inp2, alg).u) ≈ sqrt.(p)
+        @test abs.(solve(inp3, alg).u) ≈ sqrt.(p)
+        @test abs.(solve(inp4, alg).u) ≈ sqrt.(p)
+    end
+end
+
+# The following tests were included in the previos versions but these kwargs never did
+# anything!
 # # Garuntee Tests for Bisection
 # f = function (u, p)
 #     if u < 2.0
@@ -435,20 +368,3 @@ end
 # @test f(nextfloat(sol.left), nothing) >= 0.0
 # @test f(sol.right, nothing) >= 0.0
 # @test f(prevfloat(sol.right), nothing) <= 0.0
-
-# # Flipped signs & reversed tspan test for bracketing algorithms
-# f1(u, p) = u * u - p
-# f2(u, p) = p - u * u
-
-# for alg in (Alefeld(), Bisection(), Falsi(), Brent(), ITP(), Ridder())
-#     for p in 1:4
-#         inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
-#         inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
-#         inp3 = IntervalNonlinearProblem(f1, (2.0, 1.0), p)
-#         inp4 = IntervalNonlinearProblem(f2, (2.0, 1.0), p)
-#         @test abs.(solve(inp1, alg).u) ≈ sqrt.(p)
-#         @test abs.(solve(inp2, alg).u) ≈ sqrt.(p)
-#         @test abs.(solve(inp3, alg).u) ≈ sqrt.(p)
-#         @test abs.(solve(inp4, alg).u) ≈ sqrt.(p)
-#     end
-# end
diff --git a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
index 9a1989b71..9c81beb6b 100644
--- a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
@@ -6,6 +6,7 @@ p = 2.0
 vecprob = NonlinearProblem(ff, vec(u0), p)
 prob = NonlinearProblem(ff, u0, p)
 
-for alg in (Klement(), Broyden(), SimpleNewtonRaphson())
+@testset "$(alg)" for alg in (SimpleKlement(), SimpleBroyden(), SimpleNewtonRaphson(),
+    SimpleDFSane(), SimpleLimitedMemoryBroyden(), SimpleTrustRegion())
     @test vec(solve(prob, alg).u) == solve(vecprob, alg).u
 end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index a38e954d8..35a7d5c25 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -1,11 +1,12 @@
-using SafeTestsets
+using SafeTestsets, Test
 
 const GROUP = get(ENV, "GROUP", "All")
 
-@time begin
+@time @testset "SimpleNonlinearSolve.jl"
     if GROUP == "All" || GROUP == "Core"
         @time @safetestset "Basic Tests + Some AD" include("basictests.jl")
         @time @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
         @time @safetestset "Least Squares Tests" include("least_squares.jl")
+        @time @safetestset "23 Test Problems" include("23_test_problems.jl")
     end
 end

From 88e99f650f738cfee8134025bfb8f70290f44a05 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 Nov 2023 01:55:53 -0500
Subject: [PATCH 236/375] Bump version

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 8e6b0f510..3c6f51afa 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.1.26"
+version = "0.2.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From 3111c2884ef870fefce814a6645691762e01a78f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 Nov 2023 02:00:40 -0500
Subject: [PATCH 237/375] Formatting fix

---
 lib/SimpleNonlinearSolve/test/least_squares.jl | 2 +-
 lib/SimpleNonlinearSolve/test/runtests.jl      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/least_squares.jl b/lib/SimpleNonlinearSolve/test/least_squares.jl
index e09ad92bc..bc801421f 100644
--- a/lib/SimpleNonlinearSolve/test/least_squares.jl
+++ b/lib/SimpleNonlinearSolve/test/least_squares.jl
@@ -15,7 +15,7 @@ end
 prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
 
 for solver in [SimpleNewtonRaphson(AutoForwardDiff()), SimpleGaussNewton(AutoForwardDiff()),
-        SimpleNewtonRaphson(AutoFiniteDiff()), SimpleGaussNewton(AutoFiniteDiff())]
+    SimpleNewtonRaphson(AutoFiniteDiff()), SimpleGaussNewton(AutoFiniteDiff())]
     sol = solve(prob_oop, solver)
     @test norm(sol.resid) < 1e-12
 end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 35a7d5c25..cc4cd70b3 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -2,7 +2,7 @@ using SafeTestsets, Test
 
 const GROUP = get(ENV, "GROUP", "All")
 
-@time @testset "SimpleNonlinearSolve.jl"
+@time @testset "SimpleNonlinearSolve.jl" begin
     if GROUP == "All" || GROUP == "Core"
         @time @safetestset "Basic Tests + Some AD" include("basictests.jl")
         @time @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")

From f6c952fbb683ca79509035ab91e1715e29bf9cb1 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 27 Nov 2023 09:03:46 -0500
Subject: [PATCH 238/375] Fix the tests

---
 lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl    | 1 -
 lib/SimpleNonlinearSolve/test/23_test_problems.jl      | 7 ++++---
 lib/SimpleNonlinearSolve/test/basictests.jl            | 3 ++-
 lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index bf85dcfe3..b4db39691 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -96,7 +96,6 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
         fₖ₊₁ = norm(fx)^2 / T(2)
 
         # Compute the ratio of the actual to predicted reduction.
-        # @show size(H), size(δ)
         @bb Hδ = H × vec(δ)
         r = (fₖ₊₁ - fₖ) / (dot(δ', g) + dot(δ', Hδ) / T(2))
 
diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
index 5edd5710b..40b261c34 100644
--- a/lib/SimpleNonlinearSolve/test/23_test_problems.jl
+++ b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
@@ -1,4 +1,4 @@
-using SimpleNonlinearSolve, LinearAlgebra, NonlinearProblemLibrary, Test
+using SimpleNonlinearSolve, LinearAlgebra, NonlinearProblemLibrary, DiffEqBase, Test
 
 problems = NonlinearProblemLibrary.problems
 dicts = NonlinearProblemLibrary.dicts
@@ -23,7 +23,8 @@ function test_on_library(problems, dicts, alg_ops, broken_tests, ϵ = 1e-4;
                     end
                     broken = idx in broken_tests[alg] ? true : false
                     @test norm(res)≤ϵ broken=broken
-                catch
+                catch e
+                    @error e
                     broken = idx in broken_tests[alg] ? true : false
                     if broken
                         @test false broken=true
@@ -69,7 +70,7 @@ end
     alg_ops = (SimpleBroyden(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 6, 11, 12, 13, 14]
+    broken_tests[alg_ops[1]] = [1, 4, 5, 6, 11, 12, 13, 14]
 
     skip_tests = Dict(alg => Int[] for alg in alg_ops)
     skip_tests[alg_ops[1]] = [22]
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 413a6a512..e5f874569 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -156,7 +156,6 @@ end
         u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
         probN = NonlinearProblem(quadratic_f, u0, 2.0)
-        @show solve(probN, SimpleHalley(); termination_condition).u
         @test all(solve(probN, SimpleHalley(); termination_condition).u .≈ sqrt(2.0))
     end
 end
@@ -301,6 +300,7 @@ end
 end
 
 @testset "Tolerance Tests Interval Methods: $(alg)" for alg in (Bisection(), Falsi(), ITP())
+    tspan = (1.0, 20.0)
     probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
     tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
     ϵ = eps(1.0) #least possible tol for all methods
@@ -313,6 +313,7 @@ end
 end
 
 @testset "Tolerance Tests Interval Methods: $(alg)" for alg in (Ridder(), Brent())
+    tspan = (1.0, 20.0)
     probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
     tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution
     ϵ = eps(1.0) #least possible tol for all methods
diff --git a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
index 9c81beb6b..66f6a3d0c 100644
--- a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
@@ -8,5 +8,5 @@ prob = NonlinearProblem(ff, u0, p)
 
 @testset "$(alg)" for alg in (SimpleKlement(), SimpleBroyden(), SimpleNewtonRaphson(),
     SimpleDFSane(), SimpleLimitedMemoryBroyden(), SimpleTrustRegion())
-    @test vec(solve(prob, alg).u) == solve(vecprob, alg).u
+    @test vec(solve(prob, alg).u) ≈ solve(vecprob, alg).u
 end

From ab95a667324e02645758200b5e9bb9f157e983fa Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 27 Nov 2023 10:05:32 -0500
Subject: [PATCH 239/375] Add AllocCheck.jl

---
 lib/SimpleNonlinearSolve/test/Project.toml  |  1 +
 lib/SimpleNonlinearSolve/test/basictests.jl | 35 ++++++++++++++++++++-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
index b8072e68e..230ab90ea 100644
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -1,4 +1,5 @@
 [deps]
+AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index e5f874569..3643f5db3 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,4 +1,4 @@
-using BenchmarkTools, LinearSolve, SimpleNonlinearSolve, StaticArrays, Random,
+using AllocCheck, BenchmarkTools, LinearSolve, SimpleNonlinearSolve, StaticArrays, Random,
     LinearAlgebra, Test, ForwardDiff, DiffEqBase
 
 _nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x))
@@ -255,6 +255,39 @@ end
     end
 end
 
+# --- Allocation Checks ---
+
+## SimpleDFSane needs to allocate a history vector
+@testset "Allocation Checks: $(_nameof(alg))" for alg in (
+        SimpleNewtonRaphson(; autodiff = AutoForwardDiff(; chunksize = 2)),
+        SimpleHalley(; autodiff = AutoForwardDiff(; chunksize = 2)),
+        SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
+        SimpleTrustRegion(; autodiff = AutoForwardDiff(; chunksize = 2)))
+    @check_allocs nlsolve(prob, alg) = DiffEqBase.__solve(prob, alg; abstol = 1e-9)
+
+    nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
+    nlprob_sa = NonlinearProblem{false}(quadratic_f, @SVector[1.0, 1.0], 2.0)
+
+    try
+        nlsolve(nlprob_scalar, alg)
+        @test true
+    catch e
+        @error e
+        @test false
+    end
+
+    # ForwardDiff allocates for hessian since we don't propagate the chunksize
+    # SimpleLimitedMemoryBroyden needs to do views on the low rank matrices so the sizes
+    # are dynamic. This can be fixed but no without maintaining the simplicity of the code
+    try
+        nlsolve(nlprob_sa, alg)
+        @test true
+    catch e
+        @error e
+        @test false broken=(alg isa SimpleHalley || alg isa SimpleLimitedMemoryBroyden)
+    end
+end
+
 # --- Interval Nonlinear Problems ---
 
 @testset "Interval Nonlinear Problem: $(alg)" for alg in (Bisection(), Falsi(), Ridder(),

From e908f1417982bbcbdc668e63ec7804e9ff7e2a1f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 27 Nov 2023 10:15:42 -0500
Subject: [PATCH 240/375] Fix chunk size picking for StaticArrays

---
 lib/SimpleNonlinearSolve/src/utils.jl       | 14 ++++++++++++--
 lib/SimpleNonlinearSolve/test/basictests.jl |  8 +++-----
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 870b526f4..444128bf0 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -39,13 +39,23 @@ __standard_tag(::Nothing, x) = ForwardDiff.Tag(SimpleNonlinearSolveTag(), eltype
 __standard_tag(tag::ForwardDiff.Tag, _) = tag
 __standard_tag(tag, x) = ForwardDiff.Tag(tag, eltype(x))
 
+__pick_forwarddiff_chunk(x) = ForwardDiff.Chunk(length(x))
+function __pick_forwarddiff_chunk(x::StaticArray)
+    L = prod(Size(x))
+    if L ≤ ForwardDiff.DEFAULT_CHUNK_THRESHOLD
+        return ForwardDiff.Chunk{L}()
+    else
+        return ForwardDiff.Chunk{ForwardDiff.DEFAULT_CHUNK_THRESHOLD}()
+    end
+end
+
 function __get_jacobian_config(ad::AutoForwardDiff{CS}, f, x) where {CS}
-    ck = (CS === nothing || CS ≤ 0) ? ForwardDiff.Chunk(length(x)) : ForwardDiff.Chunk{CS}()
+    ck = (CS === nothing || CS ≤ 0) ? __pick_forwarddiff_chunk(x) : ForwardDiff.Chunk{CS}()
     tag = __standard_tag(ad.tag, x)
     return ForwardDiff.JacobianConfig(f, x, ck, tag)
 end
 function __get_jacobian_config(ad::AutoForwardDiff{CS}, f!, y, x) where {CS}
-    ck = (CS === nothing || CS ≤ 0) ? ForwardDiff.Chunk(length(x)) : ForwardDiff.Chunk{CS}()
+    ck = (CS === nothing || CS ≤ 0) ? __pick_forwarddiff_chunk(x) : ForwardDiff.Chunk{CS}()
     tag = __standard_tag(ad.tag, x)
     return ForwardDiff.JacobianConfig(f!, y, x, ck, tag)
 end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 3643f5db3..a9dc24c68 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -258,11 +258,9 @@ end
 # --- Allocation Checks ---
 
 ## SimpleDFSane needs to allocate a history vector
-@testset "Allocation Checks: $(_nameof(alg))" for alg in (
-        SimpleNewtonRaphson(; autodiff = AutoForwardDiff(; chunksize = 2)),
-        SimpleHalley(; autodiff = AutoForwardDiff(; chunksize = 2)),
-        SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
-        SimpleTrustRegion(; autodiff = AutoForwardDiff(; chunksize = 2)))
+@testset "Allocation Checks: $(_nameof(alg))" for alg in ( SimpleNewtonRaphson(),
+        SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
+        SimpleTrustRegion())
     @check_allocs nlsolve(prob, alg) = DiffEqBase.__solve(prob, alg; abstol = 1e-9)
 
     nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)

From 23779dfa00e5c6f2f24ead25b5634665f3d0b348 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 27 Nov 2023 21:33:22 -0500
Subject: [PATCH 241/375] Skip 1 broyden test

---
 lib/SimpleNonlinearSolve/test/23_test_problems.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
index 40b261c34..a0d5bc68e 100644
--- a/lib/SimpleNonlinearSolve/test/23_test_problems.jl
+++ b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
@@ -73,7 +73,7 @@ end
     broken_tests[alg_ops[1]] = [1, 4, 5, 6, 11, 12, 13, 14]
 
     skip_tests = Dict(alg => Int[] for alg in alg_ops)
-    skip_tests[alg_ops[1]] = [22]
+    skip_tests[alg_ops[1]] = [2, 22]
 
     test_on_library(problems, dicts, alg_ops, broken_tests; skip_tests)
 end

From 32c20f1f7a85e843cae56c0813644870054a7a46 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 27 Nov 2023 21:56:22 -0500
Subject: [PATCH 242/375] Change the norm

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 3 ++-
 lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl       | 8 ++++----
 lib/SimpleNonlinearSolve/test/basictests.jl          | 6 +++---
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 66d7d4271..a723b0a12 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -8,7 +8,8 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
 
     import DiffEqBase: AbstractNonlinearTerminationMode,
         AbstractSafeNonlinearTerminationMode, AbstractSafeBestNonlinearTerminationMode,
-        NonlinearSafeTerminationReturnCode, get_termination_mode
+        NonlinearSafeTerminationReturnCode, get_termination_mode,
+        NONLINEARSOLVE_DEFAULT_NORM
     using FiniteDiff, ForwardDiff
     import ForwardDiff: Dual
     import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 77ee497a3..2cbbd163f 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -69,7 +69,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
 
-    fx_norm = norm(fx)^nexp
+    fx_norm = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
     α_1 = one(T)
     f_1 = fx_norm
 
@@ -99,7 +99,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
         @bb @. x += α_p * d
 
         fx = __eval_f(prob, fx, x)
-        fx_norm_new = norm(fx)^nexp
+        fx_norm_new = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
 
         while k < maxiters
             fx_norm_new ≤ (f_bar + η - γ * α_p^2 * fx_norm) && break
@@ -108,7 +108,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
             @bb @. x -= α_m * d
 
             fx = __eval_f(prob, fx, x)
-            fx_norm_new = norm(fx)^nexp
+            fx_norm_new = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
 
             fx_norm_new ≤ (f_bar + η - γ * α_m^2 * fx_norm) && break
 
@@ -118,7 +118,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
             @bb @. x += α_p * d
 
             fx = __eval_f(prob, fx, x)
-            fx_norm_new = norm(fx)^nexp
+            fx_norm_new = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
 
             k += 1
         end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index a9dc24c68..6b7040308 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -258,9 +258,9 @@ end
 # --- Allocation Checks ---
 
 ## SimpleDFSane needs to allocate a history vector
-@testset "Allocation Checks: $(_nameof(alg))" for alg in ( SimpleNewtonRaphson(),
-        SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
-        SimpleTrustRegion())
+@testset "Allocation Checks: $(_nameof(alg))" for alg in (SimpleNewtonRaphson(),
+    SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
+    SimpleTrustRegion())
     @check_allocs nlsolve(prob, alg) = DiffEqBase.__solve(prob, alg; abstol = 1e-9)
 
     nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)

From cd2e8c990a36af2a4371c8b5952e2f987c3190f0 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 27 Nov 2023 22:38:35 -0500
Subject: [PATCH 243/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 3c6f51afa..b5838a949 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "0.2.0"
+version = "1.0.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From 7f0cedd18a36a3b590c2988c3ddbe050b978f953 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 28 Nov 2023 11:23:49 -0500
Subject: [PATCH 244/375] Add compat entries and change default norm

---
 lib/SimpleNonlinearSolve/Project.toml               |  3 +++
 lib/SimpleNonlinearSolve/src/nlsolve/halley.jl      |  4 ++++
 lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl | 12 ++++++------
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index b5838a949..7db247364 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -18,11 +18,14 @@ SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [compat]
+ADTypes = "0.2"
 ArrayInterface = "7"
+ConcreteStructs = "0.2"
 DiffEqBase = "6.126"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 LinearAlgebra = "1.9"
+MaybeInplace = "0.1"
 PrecompileTools = "1"
 Reexport = "1"
 SciMLBase = "2.7"
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 161abaed1..37379764c 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -14,6 +14,10 @@ A low-overhead implementation of Halley's Method.
 
   - `autodiff`: determines the backend used for the Hessian. Defaults to
     `AutoForwardDiff()`. Valid choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
+
+!!! warning
+
+    Inplace Problems are currently not supported by this method.
 """
 @kwdef @concrete struct SimpleHalley <: AbstractNewtonAlgorithm
     autodiff = AutoForwardDiff()
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index b4db39691..35719ff21 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -71,10 +71,10 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
         termination_condition)
 
     # Set default trust region radius if not specified by user.
-    Δₘₐₓ == 0 && (Δₘₐₓ = max(norm(fx), maximum(x) - minimum(x)))
+    Δₘₐₓ == 0 && (Δₘₐₓ = max(NONLINEARSOLVE_DEFAULT_NORM(fx), maximum(x) - minimum(x)))
     Δ == 0 && (Δ = Δₘₐₓ / 11)
 
-    fₖ = 0.5 * norm(fx)^2
+    fₖ = 0.5 * NONLINEARSOLVE_DEFAULT_NORM(fx)^2
     H = ∇f' * ∇f
     g = _restructure(x, ∇f' * _vec(fx))
     shrink_counter = 0
@@ -93,7 +93,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
 
         fx = __eval_f(prob, fx, x)
 
-        fₖ₊₁ = norm(fx)^2 / T(2)
+        fₖ₊₁ = NONLINEARSOLVE_DEFAULT_NORM(fx)^2 / T(2)
 
         # Compute the ratio of the actual to predicted reduction.
         @bb Hδ = H × vec(δ)
@@ -120,7 +120,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
             fx, ∇f = value_and_jacobian(alg.autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
             # Update the trust region radius.
-            (r > η₃) && (norm(δ) ≈ Δ) && (Δ = min(t₂ * Δ, Δₘₐₓ))
+            (r > η₃) && (NONLINEARSOLVE_DEFAULT_NORM(δ) ≈ Δ) && (Δ = min(t₂ * Δ, Δₘₐₓ))
             fₖ = fₖ₊₁
 
             @bb H = transpose(∇f) × ∇f
@@ -138,12 +138,12 @@ function dogleg_method!!(cache, J, f, g, Δ)
     @bb δN .= _restructure(δN, J \ _vec(f))
     @bb δN .*= -1
     # Test if the full step is within the trust region.
-    (norm(δN) ≤ Δ) && return δN
+    (NONLINEARSOLVE_DEFAULT_NORM(δN) ≤ Δ) && return δN
 
     # Calcualte Cauchy point, optimum along the steepest descent direction.
     @bb δsd .= g
     @bb @. δsd *= -1
-    norm_δsd = norm(δsd)
+    norm_δsd = NONLINEARSOLVE_DEFAULT_NORM(δsd)
     if (norm_δsd ≥ Δ)
         @bb @. δsd *= Δ / norm_δsd
         return δsd

From f557f02ed49873dd85c2aad0655011f23aa734aa Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 28 Nov 2023 11:34:58 -0500
Subject: [PATCH 245/375] revert the norm change

---
 lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 35719ff21..b4db39691 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -71,10 +71,10 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
         termination_condition)
 
     # Set default trust region radius if not specified by user.
-    Δₘₐₓ == 0 && (Δₘₐₓ = max(NONLINEARSOLVE_DEFAULT_NORM(fx), maximum(x) - minimum(x)))
+    Δₘₐₓ == 0 && (Δₘₐₓ = max(norm(fx), maximum(x) - minimum(x)))
     Δ == 0 && (Δ = Δₘₐₓ / 11)
 
-    fₖ = 0.5 * NONLINEARSOLVE_DEFAULT_NORM(fx)^2
+    fₖ = 0.5 * norm(fx)^2
     H = ∇f' * ∇f
     g = _restructure(x, ∇f' * _vec(fx))
     shrink_counter = 0
@@ -93,7 +93,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
 
         fx = __eval_f(prob, fx, x)
 
-        fₖ₊₁ = NONLINEARSOLVE_DEFAULT_NORM(fx)^2 / T(2)
+        fₖ₊₁ = norm(fx)^2 / T(2)
 
         # Compute the ratio of the actual to predicted reduction.
         @bb Hδ = H × vec(δ)
@@ -120,7 +120,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
             fx, ∇f = value_and_jacobian(alg.autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
             # Update the trust region radius.
-            (r > η₃) && (NONLINEARSOLVE_DEFAULT_NORM(δ) ≈ Δ) && (Δ = min(t₂ * Δ, Δₘₐₓ))
+            (r > η₃) && (norm(δ) ≈ Δ) && (Δ = min(t₂ * Δ, Δₘₐₓ))
             fₖ = fₖ₊₁
 
             @bb H = transpose(∇f) × ∇f
@@ -138,12 +138,12 @@ function dogleg_method!!(cache, J, f, g, Δ)
     @bb δN .= _restructure(δN, J \ _vec(f))
     @bb δN .*= -1
     # Test if the full step is within the trust region.
-    (NONLINEARSOLVE_DEFAULT_NORM(δN) ≤ Δ) && return δN
+    (norm(δN) ≤ Δ) && return δN
 
     # Calcualte Cauchy point, optimum along the steepest descent direction.
     @bb δsd .= g
     @bb @. δsd *= -1
-    norm_δsd = NONLINEARSOLVE_DEFAULT_NORM(δsd)
+    norm_δsd = norm(δsd)
     if (norm_δsd ≥ Δ)
         @bb @. δsd *= Δ / norm_δsd
         return δsd

From 6fe4ed6af8376381160e5d2a03a1d0f8e733e4fe Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 28 Nov 2023 12:42:10 -0500
Subject: [PATCH 246/375] No transpose

---
 lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index b4db39691..8cad64582 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -97,7 +97,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
 
         # Compute the ratio of the actual to predicted reduction.
         @bb Hδ = H × vec(δ)
-        r = (fₖ₊₁ - fₖ) / (dot(δ', g) + dot(δ', Hδ) / T(2))
+        r = (fₖ₊₁ - fₖ) / (dot(δ, g) + dot(δ, Hδ) / T(2))
 
         # Update the trust region radius.
         if r < η₂

From 338816074ffe8fe65a69f9556b202add5ccf86e6 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 28 Nov 2023 13:34:30 -0500
Subject: [PATCH 247/375] Disambiguate conditionals with `Bool`

---
 .../src/nlsolve/trustRegion.jl                     | 14 +++++++-------
 lib/SimpleNonlinearSolve/src/utils.jl              |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 8cad64582..73f34357a 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -85,7 +85,6 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
     @bb Hδ = copy(x)
     dogleg_cache = (; δsd, δN_δsd, δN)
 
-    F = fx
     for k in 1:maxiters
         # Solve the trust region subproblem.
         δ = dogleg_method!!(dogleg_cache, ∇f, fx, g, Δ)
@@ -100,16 +99,16 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
         r = (fₖ₊₁ - fₖ) / (dot(δ, g) + dot(δ, Hδ) / T(2))
 
         # Update the trust region radius.
-        if r < η₂
+        if Bool(r ≥ η₂)
+            shrink_counter = 0
+        else
             Δ = t₁ * Δ
             shrink_counter += 1
             shrink_counter > max_shrink_times && return build_solution(prob, alg, x, fx;
                 retcode = ReturnCode.ConvergenceFailure)
-        else
-            shrink_counter = 0
         end
 
-        if r > η₁
+        if Bool(r ≥ η₁)
             # Termination Checks
             tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
             tc_sol !== nothing && return tc_sol
@@ -138,13 +137,14 @@ function dogleg_method!!(cache, J, f, g, Δ)
     @bb δN .= _restructure(δN, J \ _vec(f))
     @bb δN .*= -1
     # Test if the full step is within the trust region.
-    (norm(δN) ≤ Δ) && return δN
+    Bool(norm(δN) ≤ Δ) && return δN
 
     # Calcualte Cauchy point, optimum along the steepest descent direction.
     @bb δsd .= g
     @bb @. δsd *= -1
     norm_δsd = norm(δsd)
-    if (norm_δsd ≥ Δ)
+
+    if Bool(norm_δsd ≥ Δ)
         @bb @. δsd *= Δ / norm_δsd
         return δsd
     end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 444128bf0..b5381f33e 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -286,14 +286,14 @@ function check_termination(tc_cache, fx, x, xo, prob, alg)
 end
 function check_termination(tc_cache, fx, x, xo, prob, alg,
         ::AbstractNonlinearTerminationMode)
-    if tc_cache(fx, x, xo)
+    if Bool(tc_cache(fx, x, xo))
         return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
     end
     return nothing
 end
 function check_termination(tc_cache, fx, x, xo, prob, alg,
         ::AbstractSafeNonlinearTerminationMode)
-    if tc_cache(fx, x, xo)
+    if Bool(tc_cache(fx, x, xo))
         if tc_cache.retcode == NonlinearSafeTerminationReturnCode.Success
             retcode = ReturnCode.Success
         elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.PatienceTermination
@@ -309,7 +309,7 @@ function check_termination(tc_cache, fx, x, xo, prob, alg,
 end
 function check_termination(tc_cache, fx, x, xo, prob, alg,
         ::AbstractSafeBestNonlinearTerminationMode)
-    if tc_cache(fx, x, xo)
+    if Bool(tc_cache(fx, x, xo))
         if tc_cache.retcode == NonlinearSafeTerminationReturnCode.Success
             retcode = ReturnCode.Success
         elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.PatienceTermination

From d64875d370fd1106bf3034cbec9c19f5faa97cb4 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 28 Nov 2023 14:47:44 -0500
Subject: [PATCH 248/375] Don't support Batched TR for correctness reasons

---
 lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl      | 4 ++--
 lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 2cbbd163f..c3f00cd28 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -102,7 +102,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
         fx_norm_new = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
 
         while k < maxiters
-            fx_norm_new ≤ (f_bar + η - γ * α_p^2 * fx_norm) && break
+            Bool(fx_norm_new ≤ (f_bar + η - γ * α_p^2 * fx_norm)) && break
 
             α_p = α_p^2 * fx_norm / (fx_norm_new + (T(2) * α_p - T(1)) * fx_norm)
             @bb @. x -= α_m * d
@@ -110,7 +110,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
             fx = __eval_f(prob, fx, x)
             fx_norm_new = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
 
-            fx_norm_new ≤ (f_bar + η - γ * α_m^2 * fx_norm) && break
+            Bool(fx_norm_new ≤ (f_bar + η - γ * α_m^2 * fx_norm)) && break
 
             α_tm = α_m^2 * fx_norm / (fx_norm_new + (T(2) * α_m - T(1)) * fx_norm)
             α_p = clamp(α_p, τ_min * α_p, τ_max * α_p)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 73f34357a..3976380b4 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -99,7 +99,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
         r = (fₖ₊₁ - fₖ) / (dot(δ, g) + dot(δ, Hδ) / T(2))
 
         # Update the trust region radius.
-        if Bool(r ≥ η₂)
+        if r ≥ η₂
             shrink_counter = 0
         else
             Δ = t₁ * Δ
@@ -108,7 +108,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
                 retcode = ReturnCode.ConvergenceFailure)
         end
 
-        if Bool(r ≥ η₁)
+        if r ≥ η₁
             # Termination Checks
             tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
             tc_sol !== nothing && return tc_sol
@@ -137,14 +137,14 @@ function dogleg_method!!(cache, J, f, g, Δ)
     @bb δN .= _restructure(δN, J \ _vec(f))
     @bb δN .*= -1
     # Test if the full step is within the trust region.
-    Bool(norm(δN) ≤ Δ) && return δN
+    (norm(δN) ≤ Δ) && return δN
 
     # Calcualte Cauchy point, optimum along the steepest descent direction.
     @bb δsd .= g
     @bb @. δsd *= -1
     norm_δsd = norm(δsd)
 
-    if Bool(norm_δsd ≥ Δ)
+    if (norm_δsd ≥ Δ)
         @bb @. δsd *= Δ / norm_δsd
         return δsd
     end

From 91304a756d2fdd5b4b5bb4d56ea8d4d95b7f0b20 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 30 Nov 2023 18:40:03 -0500
Subject: [PATCH 249/375] Needs a safety copy

---
 lib/SimpleNonlinearSolve/src/nlsolve/klement.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index 5041dc4df..b4349425a 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -9,7 +9,7 @@ struct SimpleKlement <: AbstractSimpleNonlinearSolveAlgorithm end
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         termination_condition = nothing, kwargs...)
-    x = float(prob.u0)
+    @bb x = copy(float(prob.u0))
     T = eltype(x)
     fx = _get_fx(prob, x)
 

From a7847537f9a4299baa6714b00a92d41104b1e1b3 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 30 Nov 2023 20:10:32 -0500
Subject: [PATCH 250/375] Extra cache in DFSane

---
 lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index c3f00cd28..f2cc13d6b 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -78,9 +78,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
     # Generate the cache
     @bb d = copy(x)
     @bb xo = copy(x)
-    @bb x_cache = copy(x)
     @bb δx = copy(x)
-    @bb fxo = copy(fx)
     @bb δf = copy(fx)
 
     k = 0
@@ -128,13 +126,13 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
 
         # Update spectral parameter
         @bb @. δx = x - xo
-        @bb @. δf = fx - fxo
+        @bb @. δf = fx - δf
 
         σ_k = dot(δx, δx) / dot(δx, δf)
 
         # Take step
         @bb copyto!(xo, x)
-        @bb copyto!(fxo, fx)
+        @bb copyto!(δf, fx)
         fx_norm = fx_norm_new
 
         # Store function value

From 6a7e3cee04a95dc001a9ce8a59f4fca62ee59225 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 4 Dec 2023 15:20:23 -0500
Subject: [PATCH 251/375] Bug fixes

---
 .../src/nlsolve/broyden.jl                    |  4 +--
 .../src/nlsolve/dfsane.jl                     | 25 +++++++++-------
 .../src/nlsolve/halley.jl                     |  4 +--
 .../src/nlsolve/klement.jl                    | 30 ++++++++-----------
 .../src/nlsolve/lbroyden.jl                   |  4 +--
 .../src/nlsolve/raphson.jl                    |  8 ++---
 .../src/nlsolve/trustRegion.jl                |  4 +--
 lib/SimpleNonlinearSolve/src/utils.jl         |  8 +++++
 .../test/23_test_problems.jl                  |  4 +--
 9 files changed, 47 insertions(+), 44 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index 1e544a4d2..5f2dccdc7 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -7,9 +7,9 @@ and static array problems.
 struct SimpleBroyden <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000,
+        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
         termination_condition = nothing, kwargs...)
-    @bb x = copy(float(prob.u0))
+    x = __maybe_unaliased(prob.u0, alias_u0)
     fx = _get_fx(prob, x)
 
     @bb xo = copy(x)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index f2cc13d6b..4f75cf9e6 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -51,9 +51,9 @@ Computation, 75, 1429-1448.](https://www.researchgate.net/publication/220576479_
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000,
+        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
         termination_condition = nothing, kwargs...)
-    x = float(copy(prob.u0))
+    x = __maybe_unaliased(prob.u0, alias_u0)
     fx = _get_fx(prob, x)
     T = eltype(x)
 
@@ -76,6 +76,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
     history_f_k = fill(fx_norm, M)
 
     # Generate the cache
+    @bb x_cache = similar(x)
     @bb d = copy(x)
     @bb xo = copy(x)
     @bb δx = copy(x)
@@ -89,38 +90,40 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
         # Line search direction
         @bb @. d = -σ_k * fx
 
-        η = η_strategy(f_1, k, x, fx)
+        η = η_strategy(f_1, k + 1, x, fx)
         f_bar = maximum(history_f_k)
         α_p = α_1
         α_m = α_1
 
-        @bb @. x += α_p * d
+        @bb @. x_cache = x + α_p * d
 
-        fx = __eval_f(prob, fx, x)
+        fx = __eval_f(prob, fx, x_cache)
         fx_norm_new = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
 
         while k < maxiters
             Bool(fx_norm_new ≤ (f_bar + η - γ * α_p^2 * fx_norm)) && break
 
-            α_p = α_p^2 * fx_norm / (fx_norm_new + (T(2) * α_p - T(1)) * fx_norm)
-            @bb @. x -= α_m * d
+            α_tp = α_p^2 * fx_norm / (fx_norm_new + (T(2) * α_p - T(1)) * fx_norm)
+            @bb @. x_cache = x - α_m * d
 
-            fx = __eval_f(prob, fx, x)
+            fx = __eval_f(prob, fx, x_cache)
             fx_norm_new = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
 
             Bool(fx_norm_new ≤ (f_bar + η - γ * α_m^2 * fx_norm)) && break
 
             α_tm = α_m^2 * fx_norm / (fx_norm_new + (T(2) * α_m - T(1)) * fx_norm)
-            α_p = clamp(α_p, τ_min * α_p, τ_max * α_p)
+            α_p = clamp(α_tp, τ_min * α_p, τ_max * α_p)
             α_m = clamp(α_tm, τ_min * α_m, τ_max * α_m)
-            @bb @. x += α_p * d
+            @bb @. x_cache = x + α_p * d
 
-            fx = __eval_f(prob, fx, x)
+            fx = __eval_f(prob, fx, x_cache)
             fx_norm_new = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
 
             k += 1
         end
 
+        @bb copyto!(x, x_cache)
+
         tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
         tc_sol !== nothing && return tc_sol
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 37379764c..88b30a032 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -24,12 +24,12 @@ A low-overhead implementation of Halley's Method.
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000,
+        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
         termination_condition = nothing, kwargs...)
     isinplace(prob) &&
         error("SimpleHalley currently only supports out-of-place nonlinear problems")
 
-    x = copy(float(prob.u0))
+    x = __maybe_unaliased(prob.u0, alias_u0)
     fx = _get_fx(prob, x)
     T = eltype(x)
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index b4349425a..a4e6d917c 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -7,9 +7,9 @@ method is non-allocating on scalar and static array problems.
 struct SimpleKlement <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000,
+        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
         termination_condition = nothing, kwargs...)
-    @bb x = copy(float(prob.u0))
+    x = __maybe_unaliased(prob.u0, alias_u0)
     T = eltype(x)
     fx = _get_fx(prob, x)
 
@@ -21,13 +21,12 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
     @bb δx = copy(x)
     @bb fprev = copy(fx)
     @bb xo = copy(x)
-    @bb δf = copy(fx)
     @bb d = copy(x)
 
     J = __init_identity_jacobian(fx, x)
-    @bb J_cache = copy(J)
-    @bb δx² = copy(x)
-    @bb J_cache2 = copy(J)
+    @bb J_cache = similar(J)
+    @bb δx² = similar(x)
+    @bb J_cache2 = similar(J)
     @bb F = copy(J)
 
     for _ in 1:maxiters
@@ -67,23 +66,18 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
         tc_sol !== nothing && return tc_sol
 
         @bb δx .*= -1
-        @bb @. δf = fx - fprev
-
-        # Prevent division by 0
+        @bb J_cache .= J' .^ 2
         @bb @. δx² = δx^2
-        @bb @. J_cache = J^2
-        @bb d = transpose(J_cache) × vec(δx²)
-        @bb @. d = max(d, singular_tol)
-
+        @bb d = J_cache × vec(δx²)
         @bb δx² = J × vec(δx)
-        @bb @. δf = (δf - δx²) / d
-
-        _vδf, _vδx = _vec(δf), _vec(δx)
-        @bb J_cache = _vδf × transpose(_vδx)
+        @bb @. fprev = (fx - fprev - δx²) / ifelse(iszero(d), singular_tol, d)
+        @bb J_cache = vec(fprev) × transpose(_vec(δx))
         @bb @. J_cache *= J
         @bb J_cache2 = J_cache × J
-
         @bb @. J += J_cache2
+
+        @bb copyto!(fprev, fx)
+        @bb copyto!(xo, x)
     end
 
     return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 4cc8ee025..6dcac6a37 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -22,9 +22,9 @@ function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27))
 end
 
 @views function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
-        args...; abstol = nothing, reltol = nothing, maxiters = 1000,
+        args...; abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
         termination_condition = nothing, kwargs...)
-    @bb x = copy(float(prob.u0))
+    x = __maybe_unaliased(prob.u0, alias_u0)
     threshold = __get_threshold(alg)
     η = min(SciMLBase._unwrap_val(threshold), maxiters)
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
index 22f7fba84..2ae14cddf 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
@@ -24,8 +24,8 @@ const SimpleGaussNewton = SimpleNewtonRaphson
 
 function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
         alg::SimpleNewtonRaphson, args...; abstol = nothing, reltol = nothing,
-        maxiters = 1000, termination_condition = nothing, kwargs...)
-    @bb x = copy(float(prob.u0))
+        maxiters = 1000, termination_condition = nothing, alias_u0 = false, kwargs...)
+    x = __maybe_unaliased(prob.u0, alias_u0)
     fx = _get_fx(prob, x)
     @bb xo = copy(x)
     J, jac_cache = jacobian_cache(alg.autodiff, prob.f, fx, x, prob.p)
@@ -37,9 +37,7 @@ function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresPr
         fx, dfx = value_and_jacobian(alg.autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
         if i == 1
-            if iszero(fx)
-                return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-            end
+            iszero(fx) && build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
         else
             # Termination Checks
             tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 3976380b4..4f3baf37e 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -49,9 +49,9 @@ scalar and static array problems.
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000,
+        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
         termination_condition = nothing, kwargs...)
-    @bb x = copy(float(prob.u0))
+    x = __maybe_unaliased(prob.u0, alias_u0)
     T = eltype(real(x))
     Δₘₐₓ = T(alg.max_trust_radius)
     Δ = T(alg.initial_trust_radius)
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index b5381f33e..22b186255 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -335,3 +335,11 @@ end
 
 @inline __eval_f(prob, fx, x) = isinplace(prob) ? (prob.f(fx, x, prob.p); fx) :
                                 prob.f(x, prob.p)
+
+# Unalias
+@inline __maybe_unaliased(x::Union{Number, SArray}, ::Bool) = x
+@inline function __maybe_unaliased(x::AbstractArray, alias::Bool)
+    # Spend time coping iff we will mutate the array
+    (alias || !ArrayInterface.can_setindex(typeof(x))) && return x
+    return deepcopy(x)
+end
diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
index a0d5bc68e..66b82fe9b 100644
--- a/lib/SimpleNonlinearSolve/test/23_test_problems.jl
+++ b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
@@ -61,7 +61,7 @@ end
     alg_ops = (SimpleDFSane(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 7, 9, 11, 12, 13, 15, 16, 17, 21, 22]
+    broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 11, 21]
 
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
@@ -82,7 +82,7 @@ end
     alg_ops = (SimpleKlement(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 13, 19, 21, 22]
+    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 6, 7, 11, 13, 22]
 
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end

From 54acf9fd94d6bff1845b3554b4ef66314e65a13b Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 4 Dec 2023 16:17:47 -0500
Subject: [PATCH 252/375] Fix DFSane tests and Klement allocations

---
 lib/SimpleNonlinearSolve/src/nlsolve/klement.jl |  7 ++++---
 lib/SimpleNonlinearSolve/test/basictests.jl     | 10 +++++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index a4e6d917c..262d3b4c3 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -44,10 +44,11 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
             # Singularity test
             if !issuccess(F_)
                 J = __init_identity_jacobian!!(J)
+                @bb copyto!(F, J)
                 if setindex_trait(J) === CanSetindex()
-                    lu!(J; check = false)
+                    F_ = lu!(F; check = false)
                 else
-                    J = lu(J; check = false)
+                    F_ = lu(F; check = false)
                 end
             end
         end
@@ -66,7 +67,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
         tc_sol !== nothing && return tc_sol
 
         @bb δx .*= -1
-        @bb J_cache .= J' .^ 2
+        @bb J_cache .= transpose(J) .^ 2
         @bb @. δx² = δx^2
         @bb d = J_cache × vec(δx²)
         @bb δx² = J × vec(δx)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 6b7040308..26038cd8c 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -198,11 +198,11 @@ end
             res = benchmark_nlsolve_oop(quadratic_f, @SVector[1.0, 1.0], p)
 
             if any(x -> isnan(x) || x <= 1e-5 || x >= 1e5, res)
-                @test_broken all(res .≈ sqrt(p))
+                @test_broken all(abs.(res) .≈ sqrt(p))
                 @test_broken abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
                     @SVector[1.0, 1.0], p).u[end], p)) ≈ 1 / (2 * sqrt(p))
             else
-                @test all(res .≈ sqrt(p))
+                @test all(abs.(res) .≈ sqrt(p))
                 @test isapprox(abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
                         @SVector[1.0, 1.0], p).u[end], p)), 1 / (2 * sqrt(p)))
             end
@@ -213,12 +213,12 @@ end
         for p in 1.0:0.1:100.0
             res = benchmark_nlsolve_oop(quadratic_f, 1.0, p)
 
-            if any(x -> isnan(x) || x <= 1e-5 || x >= 1e5, res)
-                @test_broken all(res .≈ sqrt(p))
+            if any(x -> isnan(x), res)
+                @test_broken abs(res.u) ≈ sqrt(p)
                 @test_broken abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
                         1.0, p).u, p)) ≈ 1 / (2 * sqrt(p))
             else
-                @test all(res .≈ sqrt(p))
+                @test abs(res.u) ≈ sqrt(p)
                 @test isapprox(abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
                             1.0, p).u, p)), 1 / (2 * sqrt(p)))
             end

From b8afd8d74eb637562e007938f15c822d3bbbe95f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 4 Dec 2023 16:26:37 -0500
Subject: [PATCH 253/375] Typo

---
 lib/SimpleNonlinearSolve/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index 6bba38ff1..256a65ad4 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -40,7 +40,7 @@ sol = solve(probB, ITP())
 
 For more details on the bracketing methods, refer to the [Tutorials](https://docs.sciml.ai/NonlinearSolve/stable/tutorials/nonlinear/#Using-Bracketing-Methods) and detailed [APIs](https://docs.sciml.ai/NonlinearSolve/stable/api/simplenonlinearsolve/#Solver-API)
 
-## Breaking Changes in v2
+## Breaking Changes in v1.0.0
 
   - Batched solvers have been removed in favor of `BatchedArrays.jl`. Stay tuned for detailed
     tutorials on how to use `BatchedArrays.jl` with `NonlinearSolve` & `SimpleNonlinearSolve`

From e196db7c0fad3c29b1de14bd18b607bc664c82e4 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 6 Dec 2023 14:48:20 -0500
Subject: [PATCH 254/375] Update Klement to exploit the diagonal structure

---
 lib/SimpleNonlinearSolve/Project.toml         |  2 +-
 .../src/nlsolve/klement.jl                    | 47 ++-----------------
 lib/SimpleNonlinearSolve/src/utils.jl         |  7 +++
 .../test/23_test_problems.jl                  |  2 +-
 4 files changed, 14 insertions(+), 44 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 7db247364..66caa1e17 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.0.0"
+version = "1.0.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index 262d3b4c3..9351b45f1 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -13,8 +13,6 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
     T = eltype(x)
     fx = _get_fx(prob, x)
 
-    singular_tol = eps(T)^(2 // 3)
-
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
 
@@ -23,42 +21,14 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
     @bb xo = copy(x)
     @bb d = copy(x)
 
-    J = __init_identity_jacobian(fx, x)
-    @bb J_cache = similar(J)
+    J = one.(x)
     @bb δx² = similar(x)
-    @bb J_cache2 = similar(J)
-    @bb F = copy(J)
 
     for _ in 1:maxiters
-        if x isa Number
-            J < singular_tol && (J = __init_identity_jacobian!!(J))
-            F_ = J
-        else
-            @bb copyto!(F, J)
-            if setindex_trait(F) === CanSetindex()
-                F_ = lu!(F; check = false)
-            else
-                F_ = lu(F; check = false)
-            end
+        any(iszero, J) && (J = __init_identity_jacobian!!(J))
 
-            # Singularity test
-            if !issuccess(F_)
-                J = __init_identity_jacobian!!(J)
-                @bb copyto!(F, J)
-                if setindex_trait(J) === CanSetindex()
-                    F_ = lu!(F; check = false)
-                else
-                    F_ = lu(F; check = false)
-                end
-            end
-        end
+        @bb @. δx = fprev / J
 
-        @bb copyto!(δx, fprev)
-        if setindex_trait(δx) === CanSetindex()
-            ldiv!(F_, _vec(δx))
-        else
-            δx = _restructure(δx, F_ \ _vec(δx))
-        end
         @bb @. x = xo - δx
         fx = __eval_f(prob, fx, x)
 
@@ -67,15 +37,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
         tc_sol !== nothing && return tc_sol
 
         @bb δx .*= -1
-        @bb J_cache .= transpose(J) .^ 2
-        @bb @. δx² = δx^2
-        @bb d = J_cache × vec(δx²)
-        @bb δx² = J × vec(δx)
-        @bb @. fprev = (fx - fprev - δx²) / ifelse(iszero(d), singular_tol, d)
-        @bb J_cache = vec(fprev) × transpose(_vec(δx))
-        @bb @. J_cache *= J
-        @bb J_cache2 = J_cache × J
-        @bb @. J += J_cache2
+        @bb @. δx² = δx^2 * J^2
+        @bb @. J += (fx - fprev - J * δx) / ifelse(iszero(δx²), T(1e-5), δx²) * δx * (J^2)
 
         @bb copyto!(fprev, fx)
         @bb copyto!(xo, x)
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 22b186255..5ee5731f9 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -212,6 +212,10 @@ function __init_identity_jacobian!!(J)
     J[diagind(J)] .= one(eltype(J))
     return J
 end
+function __init_identity_jacobian!!(J::AbstractVector)
+    fill!(J, one(eltype(J)))
+    return J
+end
 function __init_identity_jacobian(u::StaticArray, fu)
     S1, S2 = length(fu), length(u)
     J = SMatrix{S1, S2, eltype(u)}(I)
@@ -220,6 +224,9 @@ end
 function __init_identity_jacobian!!(J::SMatrix{S1, S2}) where {S1, S2}
     return SMatrix{S1, S2, eltype(J)}(I)
 end
+function __init_identity_jacobian!!(J::SVector{S1}) where {S1}
+    return ones(SVector{S1, eltype(J)})
+end
 
 function __init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2},
         ::Val{threshold}) where {S1, S2, T1, T2, threshold}
diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
index 66b82fe9b..bf85967d3 100644
--- a/lib/SimpleNonlinearSolve/test/23_test_problems.jl
+++ b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
@@ -82,7 +82,7 @@ end
     alg_ops = (SimpleKlement(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 6, 7, 11, 13, 22]
+    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 6, 11, 12, 22]
 
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end

From a86117d172e1284f467677ea10bcef6e1774a9cd Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 6 Dec 2023 21:07:18 -0500
Subject: [PATCH 255/375] Fix links

---
 lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 4f75cf9e6..11f3af1d5 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -5,9 +5,7 @@
 
 A low-overhead implementation of the df-sane method for solving large-scale nonlinear
 systems of equations. For in depth information about all the parameters and the algorithm,
-see the paper: [W LaCruz, JM Martinez, and M Raydan (2006), Spectral residual mathod without
-gradient information for solving large-scale nonlinear systems of equations, Mathematics of
-Computation, 75, 1429-1448.](https://www.researchgate.net/publication/220576479_Spectral_Residual_Method_without_Gradient_Information_for_Solving_Large-Scale_Nonlinear_Systems_of_Equations)
+see the paper [1].
 
 ### Keyword Arguments
 
@@ -37,6 +35,12 @@ Computation, 75, 1429-1448.](https://www.researchgate.net/publication/220576479_
     ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and
     `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to
     ``||F||^2 / k^2``.
+
+### References
+
+[1] W LaCruz, JM Martinez, and M Raydan (2006), Spectral residual mathod without gradient
+information for solving large-scale nonlinear systems of equations, Mathematics of
+Computation, 75, 1429-1448.
 """
 @kwdef @concrete struct SimpleDFSane <: AbstractSimpleNonlinearSolveAlgorithm
     σ_min = 1e-10

From ca5d82f90ff85c7fb4d78e0acb377db0e045b542 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Wed, 6 Dec 2023 21:10:38 -0500
Subject: [PATCH 256/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 7db247364..66caa1e17 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.0.0"
+version = "1.0.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From 171cf42016fa6de90a7b12eaceb41559c73f837f Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 7 Dec 2023 12:07:37 -0500
Subject: [PATCH 257/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 66caa1e17..f1608e1ce 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.0.1"
+version = "1.0.2"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From beabc41712c94a286ece4767b2023dd928fadfb4 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 17 Dec 2023 13:56:06 -0500
Subject: [PATCH 258/375] Fix non vec Limited Memory Broyden

---
 lib/SimpleNonlinearSolve/Project.toml                  | 2 +-
 lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl       | 8 ++++----
 lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index f1608e1ce..53749a72f 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.0.2"
+version = "1.0.3"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 6dcac6a37..a78611a0a 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -68,8 +68,8 @@ end
         d = dot(vᵀ, δf)
         @bb @. δx = (δx - mvec) / d
 
-        selectdim(U, 2, mod1(i, η)) .= δx
-        selectdim(Vᵀ, 1, mod1(i, η)) .= vᵀ
+        selectdim(U, 2, mod1(i, η)) .= _vec(δx)
+        selectdim(Vᵀ, 1, mod1(i, η)) .= _vec(vᵀ)
 
         _U = selectdim(U, 2, 1:min(η, i))
         _Vᵀ = selectdim(Vᵀ, 1, 1:min(η, i))
@@ -93,7 +93,7 @@ function _rmatvec!!(y, xᵀU, U, Vᵀ, x)
     x_ = vec(x)
     xᵀU_ = view(xᵀU, 1:η)
     @bb xᵀU_ = transpose(U) × x_
-    @bb y = transpose(Vᵀ) × xᵀU_
+    @bb y = transpose(Vᵀ) × vec(xᵀU_)
     @bb @. y -= x
     return y
 end
@@ -108,7 +108,7 @@ function _matvec!!(y, Vᵀx, U, Vᵀ, x)
     x_ = vec(x)
     Vᵀx_ = view(Vᵀx, 1:η)
     @bb Vᵀx_ = Vᵀ × x_
-    @bb y = U × Vᵀx_
+    @bb y = U × vec(Vᵀx_)
     @bb @. y -= x
     return y
 end
diff --git a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
index 66f6a3d0c..455aac91a 100644
--- a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
@@ -1,12 +1,12 @@
 using SimpleNonlinearSolve
 
 ff(u, p) = u .* u .- p
-u0 = rand(2, 2)
+u0 = ones(2, 3)
 p = 2.0
 vecprob = NonlinearProblem(ff, vec(u0), p)
 prob = NonlinearProblem(ff, u0, p)
 
 @testset "$(alg)" for alg in (SimpleKlement(), SimpleBroyden(), SimpleNewtonRaphson(),
-    SimpleDFSane(), SimpleLimitedMemoryBroyden(), SimpleTrustRegion())
+    SimpleDFSane(), SimpleLimitedMemoryBroyden(; threshold = Val(2)), SimpleTrustRegion())
     @test vec(solve(prob, alg).u) ≈ solve(vecprob, alg).u
 end

From d7deef5ab0b2e6c1b3857e4cb24344d2b8637788 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 25 Dec 2023 12:22:31 -0500
Subject: [PATCH 259/375] Fix the 23 test problems

---
 lib/SimpleNonlinearSolve/test/23_test_problems.jl | 4 ++--
 lib/SimpleNonlinearSolve/test/Project.toml        | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
index bf85967d3..bc82145a9 100644
--- a/lib/SimpleNonlinearSolve/test/23_test_problems.jl
+++ b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
@@ -42,7 +42,7 @@ end
 
     # dictionary with indices of test problems where method does not converge to small residual
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [6]
+    broken_tests[alg_ops[1]] = []
 
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
@@ -82,7 +82,7 @@ end
     alg_ops = (SimpleKlement(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 6, 11, 12, 22]
+    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 11, 12, 22]
 
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
index 230ab90ea..993e98749 100644
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -11,3 +11,6 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[compat]
+NonlinearProblemLibrary = "0.1.2"

From 703d6ca3ded3c313e61caa2abce0c054b8edec8b Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 25 Dec 2023 12:31:02 -0500
Subject: [PATCH 260/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 53749a72f..8cd72d57f 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.0.3"
+version = "1.0.4"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From ff3c7bebde65ab7bfd7850d744062829336c7fa5 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 25 Dec 2023 15:33:04 -0500
Subject: [PATCH 261/375] Add Polyester ForwardDiff support

---
 lib/SimpleNonlinearSolve/Project.toml         | 10 ++++-
 ...leNonlinearSolvePolyesterForwardDiffExt.jl | 13 ++++++
 .../src/SimpleNonlinearSolve.jl               |  4 +-
 .../src/nlsolve/halley.jl                     |  7 ++-
 .../src/nlsolve/raphson.jl                    | 12 ++---
 .../src/nlsolve/trustRegion.jl                | 12 ++---
 lib/SimpleNonlinearSolve/src/utils.jl         | 45 ++++++++++++++-----
 7 files changed, 78 insertions(+), 25 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 8cd72d57f..124e21010 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.0.4"
+version = "1.1.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -17,8 +17,14 @@ Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
+[extensions]
+SimpleNonlinearSolvePolyesterForwardDiffExt = "PolyesterForwardDiff"
+
+[weakdeps]
+PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
+
 [compat]
-ADTypes = "0.2"
+ADTypes = "0.2.6"
 ArrayInterface = "7"
 ConcreteStructs = "0.2"
 DiffEqBase = "6.126"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
new file mode 100644
index 000000000..c15738f11
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
@@ -0,0 +1,13 @@
+module SimpleNonlinearSolvePolyesterForwardDiffExt
+
+using SimpleNonlinearSolve, PolyesterForwardDiff
+
+@inline SimpleNonlinearSolve.__is_extension_loaded(::Val{:PolyesterForwardDiff}) = true
+
+@inline function SimpleNonlinearSolve.__polyester_forwarddiff_jacobian!(f!::F, y, J, x,
+        chunksize) where {F}
+    PolyesterForwardDiff.threaded_jacobian!(f!, y, J, x, chunksize)
+    return J
+end
+
+end
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index a723b0a12..6ffe25420 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -9,7 +9,7 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
     import DiffEqBase: AbstractNonlinearTerminationMode,
         AbstractSafeNonlinearTerminationMode, AbstractSafeBestNonlinearTerminationMode,
         NonlinearSafeTerminationReturnCode, get_termination_mode,
-        NONLINEARSOLVE_DEFAULT_NORM
+        NONLINEARSOLVE_DEFAULT_NORM, _get_tolerance
     using FiniteDiff, ForwardDiff
     import ForwardDiff: Dual
     import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
@@ -23,6 +23,8 @@ abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorith
 abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 abstract type AbstractNewtonAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 
+@inline __is_extension_loaded(::Val) = false
+
 include("utils.jl")
 
 ## Nonlinear Solvers
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 88b30a032..fe83c0cfa 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -53,14 +53,17 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
         fx, dfx, d2fx = compute_jacobian_and_hessian(alg.autodiff, prob, fx, x)
         setindex_trait(x) === CannotSetindex() && (A = dfx)
 
-        aᵢ = dfx \ _vec(fx)
+        # Factorize Once and Reuse
+        dfx_fact = factorize(dfx)
+
+        aᵢ = dfx_fact \ _vec(fx)
         A_ = _vec(A)
         @bb A_ = d2fx × aᵢ
         A = _restructure(A, A_)
 
         @bb Aaᵢ = A × aᵢ
         @bb A .*= -1
-        bᵢ = dfx \ Aaᵢ
+        bᵢ = dfx_fact \ Aaᵢ
 
         cᵢ_ = _vec(cᵢ)
         @bb @. cᵢ_ = (aᵢ * aᵢ) / (-aᵢ + (T(0.5) * bᵢ))
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
index 2ae14cddf..e84f59521 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
@@ -1,6 +1,6 @@
 """
     SimpleNewtonRaphson(autodiff)
-    SimpleNewtonRaphson(; autodiff = AutoForwardDiff())
+    SimpleNewtonRaphson(; autodiff = nothing)
 
 A low-overhead implementation of Newton-Raphson. This method is non-allocating on scalar
 and static array problems.
@@ -14,10 +14,11 @@ and static array problems.
 ### Keyword Arguments
 
   - `autodiff`: determines the backend used for the Jacobian. Defaults to
-    `AutoForwardDiff()`. Valid choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
+    `nothing`. Valid choices are `AutoPolyesterForwardDiff()`, `AutoForwardDiff()` or
+    `AutoFiniteDiff()`.
 """
 @kwdef @concrete struct SimpleNewtonRaphson <: AbstractNewtonAlgorithm
-    autodiff = AutoForwardDiff()
+    autodiff = nothing
 end
 
 const SimpleGaussNewton = SimpleNewtonRaphson
@@ -27,14 +28,15 @@ function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresPr
         maxiters = 1000, termination_condition = nothing, alias_u0 = false, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
     fx = _get_fx(prob, x)
+    autodiff = __get_concrete_autodiff(prob, alg.autodiff)
     @bb xo = copy(x)
-    J, jac_cache = jacobian_cache(alg.autodiff, prob.f, fx, x, prob.p)
+    J, jac_cache = jacobian_cache(autodiff, prob.f, fx, x, prob.p)
 
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
 
     for i in 1:maxiters
-        fx, dfx = value_and_jacobian(alg.autodiff, prob.f, fx, x, prob.p, jac_cache; J)
+        fx, dfx = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
         if i == 1
             iszero(fx) && build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 4f3baf37e..e8a90cdbd 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -10,7 +10,8 @@ scalar and static array problems.
 ### Keyword Arguments
 
   - `autodiff`: determines the backend used for the Jacobian. Defaults to
-    `AutoForwardDiff()`. Valid choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
+    `nothing`. Valid choices are `AutoPolyesterForwardDiff()`, `AutoForwardDiff()` or
+    `AutoFiniteDiff()`.
   - `max_trust_radius`: the maximum radius of the trust region. Defaults to
     `max(norm(f(u0)), maximum(u0) - minimum(u0))`.
   - `initial_trust_radius`: the initial trust region radius. Defaults to
@@ -37,7 +38,7 @@ scalar and static array problems.
     row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`.
 """
 @kwdef @concrete struct SimpleTrustRegion <: AbstractNewtonAlgorithm
-    autodiff = AutoForwardDiff()
+    autodiff = nothing
     max_trust_radius = 0.0
     initial_trust_radius = 0.0
     step_threshold = 0.0001
@@ -61,11 +62,12 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
     t₁ = T(alg.shrink_factor)
     t₂ = T(alg.expand_factor)
     max_shrink_times = alg.max_shrink_times
+    autodiff = __get_concrete_autodiff(prob, alg.autodiff)
 
     fx = _get_fx(prob, x)
     @bb xo = copy(x)
-    J, jac_cache = jacobian_cache(alg.autodiff, prob.f, fx, x, prob.p)
-    fx, ∇f = value_and_jacobian(alg.autodiff, prob.f, fx, x, prob.p, jac_cache; J)
+    J, jac_cache = jacobian_cache(autodiff, prob.f, fx, x, prob.p)
+    fx, ∇f = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
@@ -116,7 +118,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
             # Take the step.
             @bb @. xo = x
 
-            fx, ∇f = value_and_jacobian(alg.autodiff, prob.f, fx, x, prob.p, jac_cache; J)
+            fx, ∇f = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
             # Update the trust region radius.
             (r > η₃) && (norm(δ) ≈ Δ) && (Δ = min(t₂ * Δ, Δₘₐₓ))
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 5ee5731f9..b4b21162e 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -26,15 +26,6 @@ Return the maximum of `a` and `b` if `x1 > x0`, otherwise return the minimum.
 """
 __max_tdir(a, b, x0, x1) = ifelse(x1 > x0, max(a, b), min(a, b))
 
-__cvt_real(::Type{T}, ::Nothing) where {T} = nothing
-__cvt_real(::Type{T}, x) where {T} = real(T(x))
-
-_get_tolerance(η, ::Type{T}) where {T} = __cvt_real(T, η)
-function _get_tolerance(::Nothing, ::Type{T}) where {T}
-    η = real(oneunit(T)) * (eps(real(one(T))))^(4 // 5)
-    return _get_tolerance(η, T)
-end
-
 __standard_tag(::Nothing, x) = ForwardDiff.Tag(SimpleNonlinearSolveTag(), eltype(x))
 __standard_tag(tag::ForwardDiff.Tag, _) = tag
 __standard_tag(tag, x) = ForwardDiff.Tag(tag, eltype(x))
@@ -60,6 +51,12 @@ function __get_jacobian_config(ad::AutoForwardDiff{CS}, f!, y, x) where {CS}
     return ForwardDiff.JacobianConfig(f!, y, x, ck, tag)
 end
 
+function __get_jacobian_config(ad::AutoPolyesterForwardDiff{CS}, args...) where {CS}
+    x = last(args)
+    return (CS === nothing || CS ≤ 0) ? __pick_forwarddiff_chunk(x) :
+           ForwardDiff.Chunk{CS}()
+end
+
 """
     value_and_jacobian(ad, f, y, x, p, cache; J = nothing)
 
@@ -81,6 +78,9 @@ function value_and_jacobian(ad, f::F, y, x::X, p, cache; J = nothing) where {F,
             FiniteDiff.finite_difference_jacobian!(J, _f, x, cache)
             _f(y, x)
             return y, J
+        elseif ad isa AutoPolyesterForwardDiff
+            __polyester_forwarddiff_jacobian!(_f, y, J, x, cache)
+            return y, J
         else
             throw(ArgumentError("Unsupported AD method: $(ad)"))
         end
@@ -100,12 +100,18 @@ function value_and_jacobian(ad, f::F, y, x::X, p, cache; J = nothing) where {F,
         elseif ad isa AutoFiniteDiff
             J_fd = FiniteDiff.finite_difference_jacobian(_f, x, cache)
             return _f(x), J_fd
+        elseif ad isa AutoPolyesterForwardDiff
+            __polyester_forwarddiff_jacobian!(_f, J, x, cache)
+            return _f(x), J
         else
             throw(ArgumentError("Unsupported AD method: $(ad)"))
         end
     end
 end
 
+# Declare functions
+function __polyester_forwarddiff_jacobian! end
+
 function value_and_jacobian(ad, f::F, y, x::Number, p, cache; J = nothing) where {F}
     if DiffEqBase.has_jac(f)
         return f(x, p), f.jac(x, p)
@@ -132,7 +138,7 @@ function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
         J = similar(y, length(y), length(x))
         if DiffEqBase.has_jac(f)
             return J, nothing
-        elseif ad isa AutoForwardDiff
+        elseif ad isa AutoForwardDiff || ad isa AutoPolyesterForwardDiff
             return J, __get_jacobian_config(ad, _f, y, x)
         elseif ad isa AutoFiniteDiff
             return J, FiniteDiff.JacobianCache(copy(x), copy(y), copy(y), ad.fdtype)
@@ -146,6 +152,10 @@ function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
         elseif ad isa AutoForwardDiff
             J = ArrayInterface.can_setindex(x) ? similar(y, length(y), length(x)) : nothing
             return J, __get_jacobian_config(ad, _f, x)
+        elseif ad isa AutoPolyesterForwardDiff
+            @assert ArrayInterface.can_setindex(x) "PolyesterForwardDiff requires mutable inputs."
+            J = similar(y, length(y), length(x))
+            return J, __get_jacobian_config(ad, _f, x)
         elseif ad isa AutoFiniteDiff
             return nothing, FiniteDiff.JacobianCache(copy(x), copy(y), copy(y), ad.fdtype)
         else
@@ -350,3 +360,18 @@ end
     (alias || !ArrayInterface.can_setindex(typeof(x))) && return x
     return deepcopy(x)
 end
+
+# Decide which AD backend to use
+@inline __get_concrete_autodiff(prob, ad::ADTypes.AbstractADType) = ad
+@inline function __get_concrete_autodiff(prob, ::Nothing)
+    if ForwardDiff.can_dual(eltype(prob.u0))
+        if __is_extension_loaded(Val(:PolyesterForwardDiff)) && !(prob.u0 isa Number) &&
+           ArrayInterface.can_setindex(prob.u0)
+            return AutoPolyesterForwardDiff()
+        else
+            return AutoForwardDiff()
+        end
+    else
+        return AutoFiniteDiff()
+    end
+end
\ No newline at end of file

From 31c596d7bd9a1c3c7699bf2f63ba8285d246c924 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 26 Dec 2023 14:40:55 -0500
Subject: [PATCH 262/375] Add tests

---
 ...pleNonlinearSolvePolyesterForwardDiffExt.jl |  8 +++++++-
 lib/SimpleNonlinearSolve/src/nlsolve/halley.jl |  9 +++++----
 lib/SimpleNonlinearSolve/src/utils.jl          | 18 ++++++++++++------
 lib/SimpleNonlinearSolve/test/Project.toml     |  1 +
 lib/SimpleNonlinearSolve/test/basictests.jl    | 10 ++++++----
 5 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
index c15738f11..81cee481d 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
@@ -10,4 +10,10 @@ using SimpleNonlinearSolve, PolyesterForwardDiff
     return J
 end
 
-end
\ No newline at end of file
+@inline function SimpleNonlinearSolve.__polyester_forwarddiff_jacobian!(f::F, J, x,
+        chunksize) where {F}
+    PolyesterForwardDiff.threaded_jacobian!(f, J, x, chunksize)
+    return J
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index fe83c0cfa..50f7d38d0 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -12,15 +12,15 @@ A low-overhead implementation of Halley's Method.
 
 ### Keyword Arguments
 
-  - `autodiff`: determines the backend used for the Hessian. Defaults to
-    `AutoForwardDiff()`. Valid choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
+  - `autodiff`: determines the backend used for the Hessian. Defaults to `nothing`. Valid
+    choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
 
 !!! warning
 
     Inplace Problems are currently not supported by this method.
 """
 @kwdef @concrete struct SimpleHalley <: AbstractNewtonAlgorithm
-    autodiff = AutoForwardDiff()
+    autodiff = nothing
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
@@ -33,6 +33,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
     fx = _get_fx(prob, x)
     T = eltype(x)
 
+    autodiff = __get_concrete_autodiff(prob, alg.autodiff; polyester = Val(false))
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
 
@@ -50,7 +51,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
 
     for i in 1:maxiters
         # Hessian Computation is unfortunately type unstable
-        fx, dfx, d2fx = compute_jacobian_and_hessian(alg.autodiff, prob, fx, x)
+        fx, dfx, d2fx = compute_jacobian_and_hessian(autodiff, prob, fx, x)
         setindex_trait(x) === CannotSetindex() && (A = dfx)
 
         # Factorize Once and Reuse
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index b4b21162e..4fb620a53 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -119,6 +119,11 @@ function value_and_jacobian(ad, f::F, y, x::Number, p, cache; J = nothing) where
         T = typeof(__standard_tag(ad.tag, x))
         out = f(ForwardDiff.Dual{T}(x, one(x)), p)
         return ForwardDiff.value(out), ForwardDiff.extract_derivative(T, out)
+    elseif ad isa AutoPolyesterForwardDiff
+        # Just use ForwardDiff
+        T = typeof(__standard_tag(nothing, x))
+        out = f(ForwardDiff.Dual{T}(x, one(x)), p)
+        return ForwardDiff.value(out), ForwardDiff.extract_derivative(T, out)
     elseif ad isa AutoFiniteDiff
         _f = Base.Fix2(f, p)
         return _f(x), FiniteDiff.finite_difference_derivative(_f, x, ad.fdtype)
@@ -153,7 +158,7 @@ function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
             J = ArrayInterface.can_setindex(x) ? similar(y, length(y), length(x)) : nothing
             return J, __get_jacobian_config(ad, _f, x)
         elseif ad isa AutoPolyesterForwardDiff
-            @assert ArrayInterface.can_setindex(x) "PolyesterForwardDiff requires mutable inputs."
+            @assert ArrayInterface.can_setindex(x) "PolyesterForwardDiff requires mutable inputs. Use AutoForwardDiff instead."
             J = similar(y, length(y), length(x))
             return J, __get_jacobian_config(ad, _f, x)
         elseif ad isa AutoFiniteDiff
@@ -362,11 +367,12 @@ end
 end
 
 # Decide which AD backend to use
-@inline __get_concrete_autodiff(prob, ad::ADTypes.AbstractADType) = ad
-@inline function __get_concrete_autodiff(prob, ::Nothing)
+@inline __get_concrete_autodiff(prob, ad::ADTypes.AbstractADType; kwargs...) = ad
+@inline function __get_concrete_autodiff(prob, ::Nothing; polyester::Val{P} = Val(true),
+        kwargs...) where {P}
     if ForwardDiff.can_dual(eltype(prob.u0))
-        if __is_extension_loaded(Val(:PolyesterForwardDiff)) && !(prob.u0 isa Number) &&
-           ArrayInterface.can_setindex(prob.u0)
+        if P && __is_extension_loaded(Val(:PolyesterForwardDiff)) &&
+           !(prob.u0 isa Number) && ArrayInterface.can_setindex(prob.u0)
             return AutoPolyesterForwardDiff()
         else
             return AutoForwardDiff()
@@ -374,4 +380,4 @@ end
     else
         return AutoFiniteDiff()
     end
-end
\ No newline at end of file
+end
diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
index 993e98749..4442a15a6 100644
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -7,6 +7,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 NonlinearProblemLibrary = "b7050fa9-e91f-4b37-bcee-a89a063da141"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 26038cd8c..7b4e7bbc8 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,5 +1,6 @@
 using AllocCheck, BenchmarkTools, LinearSolve, SimpleNonlinearSolve, StaticArrays, Random,
     LinearAlgebra, Test, ForwardDiff, DiffEqBase
+import PolyesterForwardDiff
 
 _nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x))
 
@@ -29,20 +30,21 @@ const TERMINATION_CONDITIONS = [
 @testset "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion)
     # Eval else the alg is type unstable
     @eval begin
-        function benchmark_nlsolve_oop(f, u0, p = 2.0; autodiff = AutoForwardDiff())
+        function benchmark_nlsolve_oop(f, u0, p = 2.0; autodiff = nothing)
             prob = NonlinearProblem{false}(f, u0, p)
             return solve(prob, $(alg)(; autodiff), abstol = 1e-9)
         end
 
-        function benchmark_nlsolve_iip(f, u0, p = 2.0; autodiff = AutoForwardDiff())
+        function benchmark_nlsolve_iip(f, u0, p = 2.0; autodiff = nothing)
             prob = NonlinearProblem{true}(f, u0, p)
             return solve(prob, $(alg)(; autodiff), abstol = 1e-9)
         end
     end
 
     @testset "AutoDiff: $(_nameof(autodiff))" for autodiff in (AutoFiniteDiff(),
-        AutoForwardDiff())
+        AutoForwardDiff(), AutoPolyesterForwardDiff())
         @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
+            u0 isa SVector && autodiff isa AutoPolyesterForwardDiff && continue
             sol = benchmark_nlsolve_oop(quadratic_f, u0; autodiff)
             @test SciMLBase.successful_retcode(sol)
             @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
@@ -103,7 +105,7 @@ end
 # --- SimpleHalley tests ---
 
 @testset "SimpleHalley" begin
-    function benchmark_nlsolve_oop(f, u0, p = 2.0; autodiff = AutoForwardDiff())
+    function benchmark_nlsolve_oop(f, u0, p = 2.0; autodiff = nothing)
         prob = NonlinearProblem{false}(f, u0, p)
         return solve(prob, SimpleHalley(; autodiff), abstol = 1e-9)
     end

From 0f8ce36158903aad36642ef8a842d7b29a23c520 Mon Sep 17 00:00:00 2001
From: ErikQQY <2283984853@qq.com>
Date: Mon, 25 Dec 2023 23:58:16 +0800
Subject: [PATCH 263/375] Add downgrade CI

Signed-off-by: ErikQQY <2283984853@qq.com>
---
 .../.github/workflows/Downgrade.yml           | 29 +++++++++++++++++++
 lib/SimpleNonlinearSolve/Project.toml         | 20 ++++++-------
 2 files changed, 39 insertions(+), 10 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
new file mode 100644
index 000000000..01ff8cad5
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
@@ -0,0 +1,29 @@
+name: Downgrade
+on:
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: ['1']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+      - uses: cjdoris/julia-downgrade-compat-action@v1
+#        if: ${{ matrix.version == '1.6' }}
+        with:
+          skip: Pkg,TOML
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 124e21010..b28d90b98 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -25,15 +25,15 @@ PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
 
 [compat]
 ADTypes = "0.2.6"
-ArrayInterface = "7"
-ConcreteStructs = "0.2"
-DiffEqBase = "6.126"
-FiniteDiff = "2"
-ForwardDiff = "0.10.3"
+ArrayInterface = "7.7"
+ConcreteStructs = "0.2.2"
+DiffEqBase = "6.144"
+FiniteDiff = "2.21"
+ForwardDiff = "0.10.36"
 LinearAlgebra = "1.9"
-MaybeInplace = "0.1"
-PrecompileTools = "1"
-Reexport = "1"
-SciMLBase = "2.7"
-StaticArraysCore = "1.4"
+MaybeInplace = "0.1.1"
+PrecompileTools = "1.2"
+Reexport = "1.2"
+SciMLBase = "2.11"
+StaticArraysCore = "1.4.2"
 julia = "1.9"

From b35a140865f42c0a2ead9314b12bae911620f9d6 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 25 Dec 2023 11:47:12 -0500
Subject: [PATCH 264/375] Update .github/workflows/Downgrade.yml

---
 lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
index 01ff8cad5..cbcc6ac06 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
@@ -2,7 +2,7 @@ name: Downgrade
 on:
   pull_request:
     branches:
-      - master
+      - main
     paths-ignore:
       - 'docs/**'
   push:

From 77cade8edfba21ac69d81b108115703d7fb0748c Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 26 Dec 2023 18:07:56 -0500
Subject: [PATCH 265/375] Add ForwardDiff Inplace Overloads

---
 lib/SimpleNonlinearSolve/Project.toml         |  2 +-
 lib/SimpleNonlinearSolve/src/ad.jl            | 95 ++++++++++--------
 .../src/nlsolve/halley.jl                     |  9 +-
 lib/SimpleNonlinearSolve/src/utils.jl         |  3 +
 lib/SimpleNonlinearSolve/test/basictests.jl   | 98 -------------------
 lib/SimpleNonlinearSolve/test/forward_ad.jl   | 93 ++++++++++++++++++
 lib/SimpleNonlinearSolve/test/runtests.jl     |  3 +-
 7 files changed, 160 insertions(+), 143 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/test/forward_ad.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 124e21010..7a7ac60d2 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.1.0"
+version = "1.2.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index d4cbcf744..f6f5f5895 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -1,21 +1,24 @@
-function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
-    f = prob.f
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, <:AbstractArray},
+            iip, <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) where {T, V, P, iip}
+    sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
+    dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
+    return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats,
+        sol.original)
+end
+
+function __nlsolve_ad(prob::NonlinearProblem{uType, iip}, alg, args...;
+        kwargs...) where {uType, iip}
     p = value(prob.p)
-    if prob isa IntervalNonlinearProblem
-        tspan = value.(prob.tspan)
-        newprob = IntervalNonlinearProblem(f, tspan, p; prob.kwargs...)
-    else
-        u0 = value(prob.u0)
-        newprob = NonlinearProblem(f, u0, p; prob.kwargs...)
-    end
+    newprob = NonlinearProblem(prob.f, value(prob.u0), p; prob.kwargs...)
 
     sol = solve(newprob, alg, args...; kwargs...)
 
     uu = sol.u
-    f_p = scalar_nlsolve_∂f_∂p(f, uu, p)
-    f_x = scalar_nlsolve_∂f_∂u(f, uu, p)
+    f_p = __nlsolve_∂f_∂p(prob, prob.f, uu, p)
+    f_x = __nlsolve_∂f_∂u(prob, prob.f, uu, p)
 
-    z_arr = -inv(f_x) * f_p
+    z_arr = -f_x \ f_p
 
     pp = prob.p
     sumfun = ((z, p),) -> map(zᵢ -> zᵢ * ForwardDiff.partials(p), z)
@@ -30,49 +33,57 @@ function scalar_nlsolve_ad(prob, alg, args...; kwargs...)
     return sol, partials
 end
 
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, SVector, <:AbstractArray},
-            false, <:Dual{T, V, P}}, alg::AbstractSimpleNonlinearSolveAlgorithm, args...;
-        kwargs...) where {T, V, P}
-    sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
-    dual_soln = scalar_nlsolve_dual_soln(sol.u, partials, prob.p)
-    return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode)
-end
-
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, SVector, <:AbstractArray},
-            false, <:AbstractArray{<:Dual{T, V, P}}},
-        alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) where {T, V, P}
-    sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
-    dual_soln = scalar_nlsolve_dual_soln(sol.u, partials, prob.p)
-    return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode)
-end
-
-function scalar_nlsolve_∂f_∂p(f, u, p)
-    ff = p isa Number ? ForwardDiff.derivative :
-         (u isa Number ? ForwardDiff.gradient : ForwardDiff.jacobian)
-    return ff(Base.Fix1(f, u), p)
+@inline function __nlsolve_∂f_∂p(prob, f::F, u, p) where {F}
+    if isinplace(prob)
+        __f = p -> begin
+            du = similar(u, promote_type(eltype(u), eltype(p)))
+            f(du, u, p)
+            return du
+        end
+    else
+        __f = Base.Fix1(f, u)
+    end
+    if p isa Number
+        return __reshape(ForwardDiff.derivative(__f, p), :, 1)
+    elseif u isa Number
+        return __reshape(ForwardDiff.gradient(__f, p), 1, :)
+    else
+        return ForwardDiff.jacobian(__f, p)
+    end
 end
 
-function scalar_nlsolve_∂f_∂u(f, u, p)
-    ff = u isa Number ? ForwardDiff.derivative : ForwardDiff.jacobian
-    return ff(Base.Fix2(f, p), u)
+@inline function __nlsolve_∂f_∂u(prob, f::F, u, p) where {F}
+    if isinplace(prob)
+        du = similar(u)
+        __f = (du, u) -> f(du, u, p)
+        ForwardDiff.jacobian(__f, du, u)
+    else
+        __f = Base.Fix2(f, p)
+        if u isa Number
+            return ForwardDiff.derivative(__f, u)
+        else
+            return ForwardDiff.jacobian(__f, u)
+        end
+    end
 end
 
-function scalar_nlsolve_dual_soln(u::Number, partials,
+@inline function __nlsolve_dual_soln(u::Number, partials,
         ::Union{<:AbstractArray{<:Dual{T, V, P}}, Dual{T, V, P}}) where {T, V, P}
     return Dual{T, V, P}(u, partials)
 end
 
-function scalar_nlsolve_dual_soln(u::AbstractArray, partials,
+@inline function __nlsolve_dual_soln(u::AbstractArray, partials,
         ::Union{<:AbstractArray{<:Dual{T, V, P}}, Dual{T, V, P}}) where {T, V, P}
-    return map(((uᵢ, pᵢ),) -> Dual{T, V, P}(uᵢ, pᵢ), zip(u, partials))
+    _partials = _restructure(u, partials)
+    return map(((uᵢ, pᵢ),) -> Dual{T, V, P}(uᵢ, pᵢ), zip(u, _partials))
 end
 
 # avoid ambiguities
 for Alg in [Bisection]
     @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
                 <:Dual{T, V, P}}, alg::$Alg, args...; kwargs...) where {uType, iip, T, V, P}
-        sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
-        dual_soln = scalar_nlsolve_dual_soln(sol.u, partials, prob.p)
+        sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
+        dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
         return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
             left = Dual{T, V, P}(sol.left, partials),
             right = Dual{T, V, P}(sol.right, partials))
@@ -80,8 +91,8 @@ for Alg in [Bisection]
     @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
                 <:AbstractArray{<:Dual{T, V, P}}}, alg::$Alg, args...;
             kwargs...) where {uType, iip, T, V, P}
-        sol, partials = scalar_nlsolve_ad(prob, alg, args...; kwargs...)
-        dual_soln = scalar_nlsolve_dual_soln(sol.u, partials, prob.p)
+        sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
+        dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
         return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
             left = Dual{T, V, P}(sol.left, partials),
             right = Dual{T, V, P}(sol.right, partials))
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 50f7d38d0..491a340e3 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -55,7 +55,14 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
         setindex_trait(x) === CannotSetindex() && (A = dfx)
 
         # Factorize Once and Reuse
-        dfx_fact = factorize(dfx)
+        dfx_fact = if dfx isa Number
+            dfx
+        else
+            fact = lu(dfx; check = false)
+            !issuccess(fact) && return build_solution(prob, alg, x, fx;
+                retcode = ReturnCode.Unstable)
+            fact
+        end
 
         aᵢ = dfx_fact \ _vec(fx)
         A_ = _vec(A)
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 4fb620a53..b3018f5ac 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -381,3 +381,6 @@ end
         return AutoFiniteDiff()
     end
 end
+
+@inline __reshape(x::Number, args...) = x
+@inline __reshape(x::AbstractArray, args...) = reshape(x, args...)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 7b4e7bbc8..e43a90761 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -64,36 +64,6 @@ const TERMINATION_CONDITIONS = [
             autodiff = AutoForwardDiff())) == 0
     end
 
-    @testset "[OOP] Immutable AD" begin
-        for p in [1.0, 100.0]
-            @test begin
-                res = benchmark_nlsolve_oop(quadratic_f, @SVector[1.0, 1.0], p)
-                res_true = sqrt(p)
-                all(res.u .≈ res_true)
-            end
-            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
-                @SVector[1.0, 1.0], p).u[end], p) ≈ 1 / (2 * sqrt(p))
-        end
-    end
-
-    @testset "[OOP] Scalar AD" begin
-        for p in 1.0:0.1:100.0
-            @test begin
-                res = benchmark_nlsolve_oop(quadratic_f, 1.0, p)
-                res_true = sqrt(p)
-                res.u ≈ res_true
-            end
-            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f, 1.0, p).u,
-                p) ≈ 1 / (2 * sqrt(p))
-        end
-    end
-
-    t = (p) -> [sqrt(p[2] / p[1])]
-    p = [0.9, 50.0]
-    @test benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u ≈ sqrt(p[2] / p[1])
-    @test ForwardDiff.jacobian(p -> [benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u],
-        p) ≈ ForwardDiff.jacobian(t, p)
-
     @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
         u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
@@ -124,36 +94,6 @@ end
             autodiff = AutoForwardDiff())) == 0
     end
 
-    @testset "[OOP] Immutable AD" begin
-        for p in [1.0, 100.0]
-            @test begin
-                res = benchmark_nlsolve_oop(quadratic_f, @SVector[1.0, 1.0], p)
-                res_true = sqrt(p)
-                all(res.u .≈ res_true)
-            end
-            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
-                @SVector[1.0, 1.0], p).u[end], p) ≈ 1 / (2 * sqrt(p))
-        end
-    end
-
-    @testset "[OOP] Scalar AD" begin
-        for p in 1.0:0.1:100.0
-            @test begin
-                res = benchmark_nlsolve_oop(quadratic_f, 1.0, p)
-                res_true = sqrt(p)
-                res.u ≈ res_true
-            end
-            @test ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f, 1.0, p).u,
-                p) ≈ 1 / (2 * sqrt(p))
-        end
-    end
-
-    t = (p) -> [sqrt(p[2] / p[1])]
-    p = [0.9, 50.0]
-    @test benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u ≈ sqrt(p[2] / p[1])
-    @test ForwardDiff.jacobian(p -> [benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u],
-        p) ≈ ForwardDiff.jacobian(t, p)
-
     @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
         u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
@@ -195,44 +135,6 @@ end
         @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, 1.0, 2.0)) == allocs
     end
 
-    @testset "[OOP] Immutable AD" begin
-        for p in [1.0, 100.0]
-            res = benchmark_nlsolve_oop(quadratic_f, @SVector[1.0, 1.0], p)
-
-            if any(x -> isnan(x) || x <= 1e-5 || x >= 1e5, res)
-                @test_broken all(abs.(res) .≈ sqrt(p))
-                @test_broken abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
-                    @SVector[1.0, 1.0], p).u[end], p)) ≈ 1 / (2 * sqrt(p))
-            else
-                @test all(abs.(res) .≈ sqrt(p))
-                @test isapprox(abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
-                        @SVector[1.0, 1.0], p).u[end], p)), 1 / (2 * sqrt(p)))
-            end
-        end
-    end
-
-    @testset "[OOP] Scalar AD" begin
-        for p in 1.0:0.1:100.0
-            res = benchmark_nlsolve_oop(quadratic_f, 1.0, p)
-
-            if any(x -> isnan(x), res)
-                @test_broken abs(res.u) ≈ sqrt(p)
-                @test_broken abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
-                        1.0, p).u, p)) ≈ 1 / (2 * sqrt(p))
-            else
-                @test abs(res.u) ≈ sqrt(p)
-                @test isapprox(abs.(ForwardDiff.derivative(p -> benchmark_nlsolve_oop(quadratic_f,
-                            1.0, p).u, p)), 1 / (2 * sqrt(p)))
-            end
-        end
-    end
-
-    t = (p) -> [sqrt(p[2] / p[1])]
-    p = [0.9, 50.0]
-    @test benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u ≈ sqrt(p[2] / p[1])
-    @test ForwardDiff.jacobian(p -> [benchmark_nlsolve_oop(quadratic_f2, 0.5, p).u],
-        p) ≈ ForwardDiff.jacobian(t, p)
-
     @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
         u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
diff --git a/lib/SimpleNonlinearSolve/test/forward_ad.jl b/lib/SimpleNonlinearSolve/test/forward_ad.jl
new file mode 100644
index 000000000..717c222df
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/forward_ad.jl
@@ -0,0 +1,93 @@
+using ForwardDiff, SimpleNonlinearSolve, StaticArrays, Test, LinearAlgebra
+
+test_f!(du, u, p) = (@. du = u^2 - p)
+test_f(u, p) = (@. u^2 - p)
+
+jacobian_f(::Number, p) = 1 / (2 * √p)
+jacobian_f(::Number, p::Number) = 1 / (2 * √p)
+jacobian_f(u, p::Number) = one.(u) .* (1 / (2 * √p))
+jacobian_f(u, p::AbstractArray) = diagm(vec(@. 1 / (2 * √p)))
+
+function solve_with(::Val{mode}, u, alg) where {mode}
+    f = if mode === :iip
+        solve_iip(p) = solve(NonlinearProblem(test_f!, u, p), alg).u
+    elseif mode === :oop
+        solve_oop(p) = solve(NonlinearProblem(test_f, u, p), alg).u
+    end
+    return f
+end
+
+__compatible(::Any, ::Val{:oop}) = true
+__compatible(::Number, ::Val{:iip}) = false
+__compatible(::AbstractArray, ::Val{:iip}) = true
+__compatible(::StaticArray, ::Val{:iip}) = false
+
+__compatible(::Any, ::Number) = true
+__compatible(::Number, ::AbstractArray) = false
+__compatible(u::AbstractArray, p::AbstractArray) = size(u) == size(p)
+
+__compatible(u::Number, ::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm) = true
+function __compatible(u::AbstractArray,
+        ::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm)
+    true
+end
+function __compatible(u::StaticArray,
+        ::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm)
+    true
+end
+
+function __compatible(::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm,
+        ::Val{:iip})
+    true
+end
+function __compatible(::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm,
+        ::Val{:oop})
+    true
+end
+__compatible(::SimpleHalley, ::Val{:iip}) = false
+
+@testset "ForwardDiff.jl Integration: $(alg)" for alg in (SimpleNewtonRaphson(),
+    SimpleTrustRegion(), SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleDFSane())
+    us = (2.0, @SVector[1.0, 1.0], [1.0, 1.0], ones(2, 2), @SArray ones(2, 2))
+
+    @testset "Scalar AD" begin
+        for p in 1.0:0.1:100.0, u0 in us, mode in (:iip, :oop)
+            __compatible(u0, alg) || continue
+            __compatible(u0, Val(mode)) || continue
+            __compatible(alg, Val(mode)) || continue
+
+            sol = solve(NonlinearProblem(test_f, u0, p), alg)
+            if SciMLBase.successful_retcode(sol)
+                gs = abs.(ForwardDiff.derivative(solve_with(Val{mode}(), u0, alg), p))
+                gs_true = abs.(jacobian_f(u0, p))
+                if !(isapprox(gs, gs_true, atol = 1e-5))
+                    @show sol.retcode, sol.u
+                    @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_gradient=gs true_gradient=gs_true
+                else
+                    @test abs.(gs)≈abs.(gs_true) atol=1e-5
+                end
+            end
+        end
+    end
+
+    @testset "Jacobian" begin
+        for u0 in us, p in ([2.0, 1.0], [2.0 1.0; 3.0 4.0]), mode in (:iip, :oop)
+            __compatible(u0, p) || continue
+            __compatible(u0, alg) || continue
+            __compatible(u0, Val(mode)) || continue
+            __compatible(alg, Val(mode)) || continue
+
+            sol = solve(NonlinearProblem(test_f, u0, p), alg)
+            if SciMLBase.successful_retcode(sol)
+                gs = abs.(ForwardDiff.jacobian(solve_with(Val{mode}(), u0, alg), p))
+                gs_true = abs.(jacobian_f(u0, p))
+                if !(isapprox(gs, gs_true, atol = 1e-5))
+                    @show sol.retcode, sol.u
+                    @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_jacobian=gs true_jacobian=gs_true
+                else
+                    @test abs.(gs)≈abs.(gs_true) atol=1e-5
+                end
+            end
+        end
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index cc4cd70b3..6cb730bc7 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -4,7 +4,8 @@ const GROUP = get(ENV, "GROUP", "All")
 
 @time @testset "SimpleNonlinearSolve.jl" begin
     if GROUP == "All" || GROUP == "Core"
-        @time @safetestset "Basic Tests + Some AD" include("basictests.jl")
+        @time @safetestset "Basic Tests" include("basictests.jl")
+        @time @safetestset "Forward AD" include("forward_ad.jl")
         @time @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
         @time @safetestset "Least Squares Tests" include("least_squares.jl")
         @time @safetestset "23 Test Problems" include("23_test_problems.jl")

From f66e91385dd2b8008b2d493c47d01673fc7b61fa Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Tue, 26 Dec 2023 19:01:44 -0500
Subject: [PATCH 266/375] Fix tests

---
 lib/SimpleNonlinearSolve/src/ad.jl            | 47 +++++++++----------
 .../src/nlsolve/halley.jl                     |  2 +-
 lib/SimpleNonlinearSolve/test/forward_ad.jl   | 23 +++------
 3 files changed, 30 insertions(+), 42 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index f6f5f5895..574904bcc 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -7,10 +7,30 @@ function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, <:AbstractArray}
         sol.original)
 end
 
-function __nlsolve_ad(prob::NonlinearProblem{uType, iip}, alg, args...;
-        kwargs...) where {uType, iip}
+# Handle Ambiguities
+for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
+    @eval begin
+        function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
+                    <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
+                alg::$(algType), args...; kwargs...) where {uType, T, V, P, iip}
+            sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
+            dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
+            return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
+                sol.stats, sol.original, left = Dual{T, V, P}(sol.left, partials),
+                right = Dual{T, V, P}(sol.right, partials))
+        end
+    end
+end
+
+function __nlsolve_ad(prob, alg, args...; kwargs...)
     p = value(prob.p)
-    newprob = NonlinearProblem(prob.f, value(prob.u0), p; prob.kwargs...)
+    if prob isa IntervalNonlinearProblem
+        tspan = value.(prob.tspan)
+        newprob = IntervalNonlinearProblem(prob.f, tspan, p; prob.kwargs...)
+    else
+        u0 = value(prob.u0)
+        newprob = NonlinearProblem(prob.f, u0, p; prob.kwargs...)
+    end
 
     sol = solve(newprob, alg, args...; kwargs...)
 
@@ -77,24 +97,3 @@ end
     _partials = _restructure(u, partials)
     return map(((uᵢ, pᵢ),) -> Dual{T, V, P}(uᵢ, pᵢ), zip(u, _partials))
 end
-
-# avoid ambiguities
-for Alg in [Bisection]
-    @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
-                <:Dual{T, V, P}}, alg::$Alg, args...; kwargs...) where {uType, iip, T, V, P}
-        sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
-        dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
-        return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
-            left = Dual{T, V, P}(sol.left, partials),
-            right = Dual{T, V, P}(sol.right, partials))
-    end
-    @eval function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
-                <:AbstractArray{<:Dual{T, V, P}}}, alg::$Alg, args...;
-            kwargs...) where {uType, iip, T, V, P}
-        sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
-        dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
-        return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
-            left = Dual{T, V, P}(sol.left, partials),
-            right = Dual{T, V, P}(sol.right, partials))
-    end
-end
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 491a340e3..44877e097 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -71,7 +71,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
 
         @bb Aaᵢ = A × aᵢ
         @bb A .*= -1
-        bᵢ = dfx_fact \ Aaᵢ
+        bᵢ = dfx_fact \ _vec(Aaᵢ)
 
         cᵢ_ = _vec(cᵢ)
         @bb @. cᵢ_ = (aᵢ * aᵢ) / (-aᵢ + (T(0.5) * bᵢ))
diff --git a/lib/SimpleNonlinearSolve/test/forward_ad.jl b/lib/SimpleNonlinearSolve/test/forward_ad.jl
index 717c222df..f545ccb0c 100644
--- a/lib/SimpleNonlinearSolve/test/forward_ad.jl
+++ b/lib/SimpleNonlinearSolve/test/forward_ad.jl
@@ -1,4 +1,5 @@
 using ForwardDiff, SimpleNonlinearSolve, StaticArrays, Test, LinearAlgebra
+import SimpleNonlinearSolve: AbstractSimpleNonlinearSolveAlgorithm
 
 test_f!(du, u, p) = (@. du = u^2 - p)
 test_f(u, p) = (@. u^2 - p)
@@ -26,24 +27,12 @@ __compatible(::Any, ::Number) = true
 __compatible(::Number, ::AbstractArray) = false
 __compatible(u::AbstractArray, p::AbstractArray) = size(u) == size(p)
 
-__compatible(u::Number, ::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm) = true
-function __compatible(u::AbstractArray,
-        ::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm)
-    true
-end
-function __compatible(u::StaticArray,
-        ::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm)
-    true
-end
+__compatible(u::Number, ::AbstractSimpleNonlinearSolveAlgorithm) = true
+__compatible(u::AbstractArray, ::AbstractSimpleNonlinearSolveAlgorithm) = true
+__compatible(u::StaticArray, ::AbstractSimpleNonlinearSolveAlgorithm) = true
 
-function __compatible(::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm,
-        ::Val{:iip})
-    true
-end
-function __compatible(::SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm,
-        ::Val{:oop})
-    true
-end
+__compatible(::AbstractSimpleNonlinearSolveAlgorithm, ::Val{:iip}) = true
+__compatible(::AbstractSimpleNonlinearSolveAlgorithm, ::Val{:oop}) = true
 __compatible(::SimpleHalley, ::Val{:iip}) = false
 
 @testset "ForwardDiff.jl Integration: $(alg)" for alg in (SimpleNewtonRaphson(),

From 5d4d76630de74109da4d7d4f2aa500219174da1b Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 12 Jan 2024 21:48:06 -0500
Subject: [PATCH 267/375] Dispatch on solve directly and forward to __solve

---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 6ffe25420..356c0b51e 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -59,6 +59,12 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing,
     return solve(prob, ITP(), args...; kwargs...)
 end
 
+# By Pass the highlevel checks for NonlinearProblem for Simple Algorithms
+function SciMLBase.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...; kwargs...)
+    return SciMLBase.__solve(prob, alg, args...; kwargs...)
+end
+
 @setup_workload begin
     for T in (Float32, Float64)
         prob_no_brack_scalar = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))

From ec1d5b30d29bcd7debec61317b1d47db4209a4fc Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 13 Jan 2024 01:12:47 -0500
Subject: [PATCH 268/375] Non allocating version of LBroyden for StaticArrays

---
 .../src/nlsolve/lbroyden.jl                   | 160 +++++++++++++++++-
 lib/SimpleNonlinearSolve/src/utils.jl         |  14 --
 2 files changed, 156 insertions(+), 18 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index a78611a0a..390db2437 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -36,7 +36,7 @@ end
 
     fx = _get_fx(prob, x)
 
-    U, Vᵀ = __init_low_rank_jacobian(x, fx, threshold)
+    U, Vᵀ = __init_low_rank_jacobian(x, fx, x isa StaticArray ? threshold : Val(η))
 
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
@@ -48,7 +48,7 @@ end
     @bb δf = copy(fx)
 
     @bb vᵀ_cache = copy(x)
-    Tcache = __lbroyden_threshold_cache(x, threshold)
+    Tcache = __lbroyden_threshold_cache(x, x isa StaticArray ? threshold : Val(η))
     @bb mat_cache = copy(x)
 
     for i in 1:maxiters
@@ -83,6 +83,105 @@ end
     return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end
 
+# Non-allocating StaticArrays version of SimpleLimitedMemoryBroyden is actually quite
+# finicky, so we'll implement it separately from the generic version
+# We make an exception here and don't support termination conditions
+@views function SciMLBase.__solve(prob::NonlinearProblem{<:SArray},
+        alg::SimpleLimitedMemoryBroyden, args...; abstol = nothing,
+        termination_condition = nothing,
+        maxiters = 1000, kwargs...)
+    if termination_condition !== nothing &&
+       !(termination_condition isa AbsNormTerminationMode)
+        error("SimpleLimitedMemoryBroyden with StaticArrays does not support termination \
+               conditions!")
+    end
+
+    x = prob.u0
+    fx = _get_fx(prob, x)
+    threshold = __get_threshold(alg)
+
+    U, Vᵀ = __init_low_rank_jacobian(x, fx, threshold)
+
+    abstol = DiffEqBase._get_tolerance(abstol, eltype(x))
+
+    xo, δx, fo, δf = x, -fx, fx, fx
+
+    converged, res = __unrolled_lbroyden_initial_iterations(prob, xo, fo, δx, abstol, U, Vᵀ,
+        threshold)
+
+    converged &&
+        return build_solution(prob, alg, res.x, res.fx; retcode = ReturnCode.Success)
+
+    xo, fo, δx = res.x, res.fx, res.δx
+
+    for i in 1:(maxiters - SciMLBase._unwrap_val(threshold))
+        x = xo .+ δx
+        fx = prob.f(x, prob.p)
+        δf = fx - fo
+
+        maximum(abs, fx) ≤ abstol &&
+            return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+
+        vᵀ = _restructure(x, _rmatvec!!(U, Vᵀ, vec(δx)))
+        mvec = _restructure(x, _matvec!!(U, Vᵀ, vec(δf)))
+
+        d = dot(vᵀ, δf)
+        δx = @. (δx - mvec) / d
+
+        U = Base.setindex(U, vec(δx), mod1(i, SciMLBase._unwrap_val(threshold)))
+        Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), mod1(i, SciMLBase._unwrap_val(threshold)))
+
+        δx = -_restructure(fx, _matvec!!(U, Vᵀ, vec(fx)))
+
+        xo = x
+        fo = fx
+    end
+
+    return build_solution(prob, alg, xo, fo; retcode = ReturnCode.MaxIters)
+end
+
+@generated function __unrolled_lbroyden_initial_iterations(prob, xo, fo, δx, abstol, U,
+        Vᵀ, ::Val{threshold}) where {threshold}
+    calls = []
+    for i in 1:threshold
+        static_idx, static_idx_p1 = Val(i - 1), Val(i)
+        push!(calls,
+            quote
+                x = xo .+ δx
+                fx = prob.f(x, prob.p)
+                δf = fx - fo
+
+                maximum(abs, fx) ≤ abstol && return true, (; x, fx, δx)
+
+                _U = __first_n_getindex(U, $(static_idx))
+                _Vᵀ = __first_n_getindex(Vᵀ, $(static_idx))
+
+                vᵀ = _restructure(x, _rmatvec!!(_U, _Vᵀ, vec(δx)))
+                mvec = _restructure(x, _matvec!!(_U, _Vᵀ, vec(δf)))
+
+                d = dot(vᵀ, δf)
+                δx = @. (δx - mvec) / d
+
+                U = Base.setindex(U, vec(δx), $(i))
+                Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), $(i))
+
+                _U = __first_n_getindex(U, $(static_idx_p1))
+                _Vᵀ = __first_n_getindex(Vᵀ, $(static_idx_p1))
+                δx = -_restructure(fx, _matvec!!(_U, _Vᵀ, vec(fx)))
+
+                xo = x
+                fo = fx
+            end)
+    end
+    push!(calls, quote
+        # Termination Check
+        maximum(abs, fx) ≤ abstol && return true, (; x, fx, δx)
+
+        return false, (; x, fx, δx)
+    end)
+    return Expr(:block, calls...)
+end
+
 function _rmatvec!!(y, xᵀU, U, Vᵀ, x)
     # xᵀ × (-I + UVᵀ)
     η = size(U, 2)
@@ -98,6 +197,9 @@ function _rmatvec!!(y, xᵀU, U, Vᵀ, x)
     return y
 end
 
+@inline _rmatvec!!(::Nothing, Vᵀ, x) = -x
+@inline _rmatvec!!(U, Vᵀ, x) = __mapTdot(__mapdot(x, U), Vᵀ) .- x
+
 function _matvec!!(y, Vᵀx, U, Vᵀ, x)
     # (-I + UVᵀ) × x
     η = size(U, 2)
@@ -113,7 +215,57 @@ function _matvec!!(y, Vᵀx, U, Vᵀ, x)
     return y
 end
 
+@inline _matvec!!(::Nothing, Vᵀ, x) = -x
+@inline _matvec!!(U, Vᵀ, x) = __mapTdot(__mapdot(x, Vᵀ), U) .- x
+
+function __mapdot(x::SVector{S1}, Y::SVector{S2, <:SVector{S1}}) where {S1, S2}
+    return map(Base.Fix1(dot, x), Y)
+end
+@generated function __mapTdot(x::SVector{S1}, Y::SVector{S1, <:SVector{S2}}) where {S1, S2}
+    calls = []
+    syms = [gensym("m$(i)") for i in 1:length(Y)]
+    for i in 1:length(Y)
+        push!(calls, :($(syms[i]) = x[$(i)] .* Y[$i]))
+    end
+    push!(calls, :(return .+($(syms...))))
+    return Expr(:block, calls...)
+end
+
+@generated function __first_n_getindex(x::SVector{L, T}, ::Val{N}) where {L, T, N}
+    @assert N ≤ L
+    getcalls = ntuple(i -> :(x[$i]), N)
+    N == 0 && return :(return nothing)
+    return :(return SVector{$N, $T}(($(getcalls...))))
+end
+
 __lbroyden_threshold_cache(x, ::Val{threshold}) where {threshold} = similar(x, threshold)
-function __lbroyden_threshold_cache(x::SArray, ::Val{threshold}) where {threshold}
-    return SArray{Tuple{threshold}, eltype(x)}(ntuple(_ -> zero(eltype(x)), threshold))
+function __lbroyden_threshold_cache(x::StaticArray, ::Val{threshold}) where {threshold}
+    return zeros(MArray{Tuple{threshold}, eltype(x)})
+end
+__lbroyden_threshold_cache(x::SArray, ::Val{threshold}) where {threshold} = nothing
+
+function __init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2},
+        ::Val{threshold}) where {S1, S2, T1, T2, threshold}
+    T = promote_type(T1, T2)
+    fuSize, uSize = Size(fu), Size(u)
+    Vᵀ = MArray{Tuple{threshold, prod(uSize)}, T}(undef)
+    U = MArray{Tuple{prod(fuSize), threshold}, T}(undef)
+    return U, Vᵀ
+end
+@generated function __init_low_rank_jacobian(u::SArray{S1, T1}, fu::SArray{S2, T2},
+        ::Val{threshold}) where {S1, S2, T1, T2, threshold}
+    T = promote_type(T1, T2)
+    Lfu, Lu = prod(Size(fu)), prod(Size(u))
+    inner_inits_Vᵀ = [zeros(SVector{Lu, T}) for i in 1:threshold]
+    inner_inits_U = [zeros(SVector{Lfu, T}) for i in 1:threshold]
+    return quote
+        Vᵀ = SVector($(inner_inits_Vᵀ...))
+        U = SVector($(inner_inits_U...))
+        return U, Vᵀ
+    end
+end
+function __init_low_rank_jacobian(u, fu, ::Val{threshold}) where {threshold}
+    Vᵀ = similar(u, threshold, length(u))
+    U = similar(u, length(fu), threshold)
+    return U, Vᵀ
 end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index b3018f5ac..c6b9f2004 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -243,20 +243,6 @@ function __init_identity_jacobian!!(J::SVector{S1}) where {S1}
     return ones(SVector{S1, eltype(J)})
 end
 
-function __init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2},
-        ::Val{threshold}) where {S1, S2, T1, T2, threshold}
-    T = promote_type(T1, T2)
-    fuSize, uSize = Size(fu), Size(u)
-    Vᵀ = MArray{Tuple{threshold, prod(uSize)}, T}(undef)
-    U = MArray{Tuple{prod(fuSize), threshold}, T}(undef)
-    return U, Vᵀ
-end
-function __init_low_rank_jacobian(u, fu, ::Val{threshold}) where {threshold}
-    Vᵀ = similar(u, threshold, length(u))
-    U = similar(u, length(fu), threshold)
-    return U, Vᵀ
-end
-
 @inline _vec(v) = vec(v)
 @inline _vec(v::Number) = v
 @inline _vec(v::AbstractVector) = v

From ebfcffd4548729c5ce1c79348a2f37a47687697c Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 13 Jan 2024 01:44:43 -0500
Subject: [PATCH 269/375] Allow allocations for the termination condition
 version

---
 .../.buildkite/pipeline.yml                   | 15 ++++++++
 .../src/nlsolve/lbroyden.jl                   | 31 +++++++++-------
 lib/SimpleNonlinearSolve/test/cuda.jl         | 35 +++++++++++++++++++
 .../test/cuda/Project.toml                    |  5 +++
 lib/SimpleNonlinearSolve/test/runtests.jl     | 11 ++++++
 5 files changed, 85 insertions(+), 12 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
 create mode 100644 lib/SimpleNonlinearSolve/test/cuda.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/cuda/Project.toml

diff --git a/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
new file mode 100644
index 000000000..5705ed5ab
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
@@ -0,0 +1,15 @@
+steps:
+  - label: "Julia 1"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1"
+    agents:
+      queue: "juliagpu"
+      cuda: "*"
+    timeout_in_minutes: 30
+    # Don't run Buildkite if the commit message includes the text [skip tests]
+    if: build.message !~ /\[skip tests\]/
+
+env:
+  GROUP: CUDA
+  JULIA_PKG_SERVER: ""
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 390db2437..fab444329 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -21,7 +21,22 @@ function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27))
     return SimpleLimitedMemoryBroyden{SciMLBase._unwrap_val(threshold)}()
 end
 
-@views function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
+        args...; termination_condition = nothing, kwargs...)
+    if prob.u0 isa SArray
+        if termination_condition === nothing ||
+           termination_condition isa AbsNormTerminationMode
+            return __static_solve(prob, alg, args...; termination_condition, kwargs...)
+        end
+        @warn "Specifying `termination_condition = $(termination_condition)` for \
+               `SimpleLimitedMemoryBroyden` with `SArray` is not non-allocating. Use \
+               either `termination_condition = AbsNormTerminationMode()` or \
+               `termination_condition = nothing`." maxlog=1
+    end
+    return __generic_solve(prob, alg, args...; termination_condition, kwargs...)
+end
+
+@views function __generic_solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
         args...; abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
         termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
@@ -85,17 +100,9 @@ end
 
 # Non-allocating StaticArrays version of SimpleLimitedMemoryBroyden is actually quite
 # finicky, so we'll implement it separately from the generic version
-# We make an exception here and don't support termination conditions
-@views function SciMLBase.__solve(prob::NonlinearProblem{<:SArray},
-        alg::SimpleLimitedMemoryBroyden, args...; abstol = nothing,
-        termination_condition = nothing,
-        maxiters = 1000, kwargs...)
-    if termination_condition !== nothing &&
-       !(termination_condition isa AbsNormTerminationMode)
-        error("SimpleLimitedMemoryBroyden with StaticArrays does not support termination \
-               conditions!")
-    end
-
+# Ignore termination_condition. Don't pass things into internal functions
+function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemoryBroyden,
+        args...; abstol = nothing, maxiters = 1000, kwargs...)
     x = prob.u0
     fx = _get_fx(prob, x)
     threshold = __get_threshold(alg)
diff --git a/lib/SimpleNonlinearSolve/test/cuda.jl b/lib/SimpleNonlinearSolve/test/cuda.jl
new file mode 100644
index 000000000..f34ec5914
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/cuda.jl
@@ -0,0 +1,35 @@
+using SimpleNonlinearSolve, StaticArrays, CUDA, Test
+
+CUDA.allowscalar(false)
+
+f(u, p) = u .* u .- 2
+f!(du, u, p) = du .= u .* u .- 2
+
+@testset "Solving on GPUs" begin
+    for alg in (SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(), SimpleBroyden(),
+        SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley())
+        @info "Testing $alg on CUDA"
+
+        # Static Arrays
+        u0 = @SVector[1.0f0, 1.0f0]
+        probN = NonlinearProblem{false}(f, u0)
+        sol = solve(probN, alg; abstol = 1.0f-6)
+        @test SciMLBase.successful_retcode(sol)
+        @test maximum(abs, sol.resid) ≤ 1.0f-6
+
+        # Regular Arrays
+        u0 = [1.0, 1.0]
+        probN = NonlinearProblem{false}(f, u0)
+        sol = solve(probN, alg; abstol = 1.0f-6)
+        @test SciMLBase.successful_retcode(sol)
+        @test maximum(abs, sol.resid) ≤ 1.0f-6
+
+        # Regular Arrays Inplace
+        alg isa SimpleHalley && continue
+        u0 = [1.0, 1.0]
+        probN = NonlinearProblem{true}(f!, u0)
+        sol = solve(probN, alg; abstol = 1.0f-6)
+        @test SciMLBase.successful_retcode(sol)
+        @test maximum(abs, sol.resid) ≤ 1.0f-6
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/cuda/Project.toml b/lib/SimpleNonlinearSolve/test/cuda/Project.toml
new file mode 100644
index 000000000..36c63059b
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/cuda/Project.toml
@@ -0,0 +1,5 @@
+[deps]
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 6cb730bc7..cfd91f8b6 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -2,6 +2,12 @@ using SafeTestsets, Test
 
 const GROUP = get(ENV, "GROUP", "All")
 
+function activate_env(env)
+    Pkg.activate(env)
+    Pkg.develop(PackageSpec(path = dirname(@__DIR__)))
+    Pkg.instantiate()
+end
+
 @time @testset "SimpleNonlinearSolve.jl" begin
     if GROUP == "All" || GROUP == "Core"
         @time @safetestset "Basic Tests" include("basictests.jl")
@@ -10,4 +16,9 @@ const GROUP = get(ENV, "GROUP", "All")
         @time @safetestset "Least Squares Tests" include("least_squares.jl")
         @time @safetestset "23 Test Problems" include("23_test_problems.jl")
     end
+
+    if GROUP == "CUDA"
+        activate_env("cuda")
+        @time @safetestset "CUDA Tests" include("cuda.jl")
+    end
 end

From 8ef96862d1ec067916f762dff390ec0f0048cb35 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 13 Jan 2024 01:51:41 -0500
Subject: [PATCH 270/375] AllocCheck for SimpleLimitedMemoryBroyden

---
 lib/SimpleNonlinearSolve/.buildkite/pipeline.yml | 1 +
 lib/SimpleNonlinearSolve/test/basictests.jl      | 6 ++----
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
index 5705ed5ab..e84559d00 100644
--- a/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
+++ b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
@@ -3,6 +3,7 @@ steps:
     plugins:
       - JuliaCI/julia#v1:
           version: "1"
+      - JuliaCI/julia-test#v1:
     agents:
       queue: "juliagpu"
       cuda: "*"
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index e43a90761..99aa3df40 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -165,7 +165,7 @@ end
 @testset "Allocation Checks: $(_nameof(alg))" for alg in (SimpleNewtonRaphson(),
     SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
     SimpleTrustRegion())
-    @check_allocs nlsolve(prob, alg) = DiffEqBase.__solve(prob, alg; abstol = 1e-9)
+    @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
 
     nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
     nlprob_sa = NonlinearProblem{false}(quadratic_f, @SVector[1.0, 1.0], 2.0)
@@ -179,14 +179,12 @@ end
     end
 
     # ForwardDiff allocates for hessian since we don't propagate the chunksize
-    # SimpleLimitedMemoryBroyden needs to do views on the low rank matrices so the sizes
-    # are dynamic. This can be fixed but no without maintaining the simplicity of the code
     try
         nlsolve(nlprob_sa, alg)
         @test true
     catch e
         @error e
-        @test false broken=(alg isa SimpleHalley || alg isa SimpleLimitedMemoryBroyden)
+        @test false broken=(alg isa SimpleHalley)
     end
 end
 

From 043336bf12ebc842a0c8f3f94cc353b76dcfd7d8 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 13 Jan 2024 02:36:02 -0500
Subject: [PATCH 271/375] Make DFSane non-allocating as well

---
 .../src/nlsolve/dfsane.jl                     | 45 ++++++++++++-------
 .../src/nlsolve/lbroyden.jl                   | 19 ++++----
 lib/SimpleNonlinearSolve/test/basictests.jl   |  5 ++-
 3 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 11f3af1d5..46f2e86d1 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -1,6 +1,6 @@
 """
     SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
-        M::Int = 10, γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+        M::Union{Int, Val} = Val(10), γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
         nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2)
 
 A low-overhead implementation of the df-sane method for solving large-scale nonlinear
@@ -42,21 +42,26 @@ see the paper [1].
 information for solving large-scale nonlinear systems of equations, Mathematics of
 Computation, 75, 1429-1448.
 """
-@kwdef @concrete struct SimpleDFSane <: AbstractSimpleNonlinearSolveAlgorithm
-    σ_min = 1e-10
-    σ_max = 1e10
-    σ_1 = 1.0
-    M::Int = 10
-    γ = 1e-4
-    τ_min = 0.1
-    τ_max = 0.5
-    nexp::Int = 2
-    η_strategy = (f_1, k, x, F) -> f_1 ./ k^2
+@concrete struct SimpleDFSane{M} <: AbstractSimpleNonlinearSolveAlgorithm
+    σ_min
+    σ_max
+    σ_1
+    γ
+    τ_min
+    τ_max
+    nexp::Int
+    η_strategy
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
+function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
+        M::Union{Int, Val} = Val(10), γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+        nexp::Int = 2, η_strategy::F = (f_1, k, x, F) -> f_1 ./ k^2) where {F}
+    return SimpleDFSane{SciMLBase._unwrap_val(M)}(σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp, η_strategy)
+end
+
+function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
-        termination_condition = nothing, kwargs...)
+        termination_condition = nothing, kwargs...) where {M}
     x = __maybe_unaliased(prob.u0, alias_u0)
     fx = _get_fx(prob, x)
     T = eltype(x)
@@ -65,7 +70,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
     σ_max = T(alg.σ_max)
     σ_k = T(alg.σ_1)
 
-    (; M, nexp, η_strategy) = alg
+    (; nexp, η_strategy) = alg
     γ = T(alg.γ)
     τ_min = T(alg.τ_min)
     τ_max = T(alg.τ_max)
@@ -77,7 +82,11 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
     α_1 = one(T)
     f_1 = fx_norm
 
-    history_f_k = fill(fx_norm, M)
+    history_f_k = if x isa SArray
+        ones(SVector{M, T}) * fx_norm
+    else
+        fill(fx_norm, M)
+    end
 
     # Generate the cache
     @bb x_cache = similar(x)
@@ -143,7 +152,11 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane, args...;
         fx_norm = fx_norm_new
 
         # Store function value
-        history_f_k[mod1(k, M)] = fx_norm_new
+        if history_f_k isa SVector
+            history_f_k = Base.setindex(history_f_k, fx_norm_new, mod1(k, M))
+        else
+            history_f_k[mod1(k, M)] = fx_norm_new
+        end
         k += 1
     end
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index fab444329..a919f5cab 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -107,7 +107,7 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
     fx = _get_fx(prob, x)
     threshold = __get_threshold(alg)
 
-    U, Vᵀ = __init_low_rank_jacobian(x, fx, threshold)
+    U, Vᵀ = __init_low_rank_jacobian(vec(x), vec(fx), threshold)
 
     abstol = DiffEqBase._get_tolerance(abstol, eltype(x))
 
@@ -230,8 +230,8 @@ function __mapdot(x::SVector{S1}, Y::SVector{S2, <:SVector{S1}}) where {S1, S2}
 end
 @generated function __mapTdot(x::SVector{S1}, Y::SVector{S1, <:SVector{S2}}) where {S1, S2}
     calls = []
-    syms = [gensym("m$(i)") for i in 1:length(Y)]
-    for i in 1:length(Y)
+    syms = [gensym("m$(i)") for i in 1:S1]
+    for i in 1:S1
         push!(calls, :($(syms[i]) = x[$(i)] .* Y[$i]))
     end
     push!(calls, :(return .+($(syms...))))
@@ -259,18 +259,21 @@ function __init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2
     U = MArray{Tuple{prod(fuSize), threshold}, T}(undef)
     return U, Vᵀ
 end
-@generated function __init_low_rank_jacobian(u::SArray{S1, T1}, fu::SArray{S2, T2},
-        ::Val{threshold}) where {S1, S2, T1, T2, threshold}
+
+@generated function __init_low_rank_jacobian(u::SVector{Lu, T1}, fu::SVector{Lfu, T2},
+        ::Val{threshold}) where {Lu, Lfu, T1, T2, threshold}
     T = promote_type(T1, T2)
-    Lfu, Lu = prod(Size(fu)), prod(Size(u))
-    inner_inits_Vᵀ = [zeros(SVector{Lu, T}) for i in 1:threshold]
-    inner_inits_U = [zeros(SVector{Lfu, T}) for i in 1:threshold]
+    # Lfu, Lu = __prod_size(S2), __prod_size(S1)
+    # Lfu, Lu = __prod(Size(fu)), __prod(Size(u))
+    inner_inits_Vᵀ = [:(zeros(SVector{$Lu, $T})) for i in 1:threshold]
+    inner_inits_U = [:(zeros(SVector{$Lfu, $T})) for i in 1:threshold]
     return quote
         Vᵀ = SVector($(inner_inits_Vᵀ...))
         U = SVector($(inner_inits_U...))
         return U, Vᵀ
     end
 end
+
 function __init_low_rank_jacobian(u, fu, ::Val{threshold}) where {threshold}
     Vᵀ = similar(u, threshold, length(u))
     U = similar(u, length(fu), threshold)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 99aa3df40..7da944e7b 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -164,7 +164,7 @@ end
 ## SimpleDFSane needs to allocate a history vector
 @testset "Allocation Checks: $(_nameof(alg))" for alg in (SimpleNewtonRaphson(),
     SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
-    SimpleTrustRegion())
+    SimpleTrustRegion(), SimpleDFSane())
     @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
 
     nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
@@ -175,7 +175,8 @@ end
         @test true
     catch e
         @error e
-        @test false
+        # History Vector Allocates
+        @test false broken=(alg isa SimpleDFSane)
     end
 
     # ForwardDiff allocates for hessian since we don't propagate the chunksize

From 91a995d9ad7302572823f9a7caa5c8b9a335b41a Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 13 Jan 2024 02:42:28 -0500
Subject: [PATCH 272/375] Add kernel launch tests

---
 lib/SimpleNonlinearSolve/Project.toml         |  2 +-
 lib/SimpleNonlinearSolve/README.md            |  1 +
 .../src/nlsolve/dfsane.jl                     |  3 ++-
 .../src/nlsolve/lbroyden.jl                   |  4 ----
 lib/SimpleNonlinearSolve/test/cuda.jl         | 23 +++++++++++++++++++
 lib/SimpleNonlinearSolve/test/runtests.jl     |  2 +-
 6 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 7a7ac60d2..d518e1686 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.2.0"
+version = "1.2.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index 256a65ad4..3ef4868bb 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -5,6 +5,7 @@
 
 [![codecov](https://codecov.io/gh/SciML/SimpleNonlinearSolve.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/SciML/SimpleNonlinearSolve.jl)
 [![Build Status](https://github.com/SciML/SimpleNonlinearSolve.jl/workflows/CI/badge.svg)](https://github.com/SciML/SimpleNonlinearSolve.jl/actions?query=workflow%3ACI)
+[![Build status](https://badge.buildkite.com/c5f7db4f1b5e8a592514378b6fc807d934546cc7d5aa79d645.svg?branch=main)](https://buildkite.com/julialang/simplenonlinearsolve-dot-jl)
 
 [![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor%27s%20Guide-blueviolet)](https://github.com/SciML/ColPrac)
 [![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 46f2e86d1..c6111b38c 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -56,7 +56,8 @@ end
 function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
         M::Union{Int, Val} = Val(10), γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
         nexp::Int = 2, η_strategy::F = (f_1, k, x, F) -> f_1 ./ k^2) where {F}
-    return SimpleDFSane{SciMLBase._unwrap_val(M)}(σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp, η_strategy)
+    return SimpleDFSane{SciMLBase._unwrap_val(M)}(σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp,
+        η_strategy)
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...;
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index a919f5cab..3a0c11ded 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -259,12 +259,9 @@ function __init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2
     U = MArray{Tuple{prod(fuSize), threshold}, T}(undef)
     return U, Vᵀ
 end
-
 @generated function __init_low_rank_jacobian(u::SVector{Lu, T1}, fu::SVector{Lfu, T2},
         ::Val{threshold}) where {Lu, Lfu, T1, T2, threshold}
     T = promote_type(T1, T2)
-    # Lfu, Lu = __prod_size(S2), __prod_size(S1)
-    # Lfu, Lu = __prod(Size(fu)), __prod(Size(u))
     inner_inits_Vᵀ = [:(zeros(SVector{$Lu, $T})) for i in 1:threshold]
     inner_inits_U = [:(zeros(SVector{$Lfu, $T})) for i in 1:threshold]
     return quote
@@ -273,7 +270,6 @@ end
         return U, Vᵀ
     end
 end
-
 function __init_low_rank_jacobian(u, fu, ::Val{threshold}) where {threshold}
     Vᵀ = similar(u, threshold, length(u))
     U = similar(u, length(fu), threshold)
diff --git a/lib/SimpleNonlinearSolve/test/cuda.jl b/lib/SimpleNonlinearSolve/test/cuda.jl
index f34ec5914..9b984bc70 100644
--- a/lib/SimpleNonlinearSolve/test/cuda.jl
+++ b/lib/SimpleNonlinearSolve/test/cuda.jl
@@ -33,3 +33,26 @@ f!(du, u, p) = du .= u .* u .- 2
         @test maximum(abs, sol.resid) ≤ 1.0f-6
     end
 end
+
+function kernel_function(prob, alg)
+    solve(prob, alg; abstol = 1.0f-6, reltol = 1.0f-6)
+    return nothing
+end
+
+@testset "CUDA Kernel Launch Test" begin
+    prob = NonlinearProblem{false}(f, @SVector[1.0f0, 1.0f0])
+
+    for alg in (SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(), SimpleBroyden(),
+        SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley())
+        @test begin
+            try
+                @cuda kernel_function(prob, alg)
+                @info "Successfully launched kernel for $(alg)."
+                true
+            catch err
+                @error "Kernel Launch failed for $(alg)."
+                false
+            end
+        end broken=(alg isa SimpleHalley)
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index cfd91f8b6..4de514642 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -1,4 +1,4 @@
-using SafeTestsets, Test
+using Pkg, SafeTestsets, Test
 
 const GROUP = get(ENV, "GROUP", "All")
 

From 686c1cb372c71f12ef4edaba4c9a67d4f133ea67 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 10 Jan 2024 09:46:10 -0500
Subject: [PATCH 273/375] Add Line Search to Broyden

---
 lib/SimpleNonlinearSolve/Project.toml         | 10 ++-
 .../src/SimpleNonlinearSolve.jl               |  6 +-
 lib/SimpleNonlinearSolve/src/linesearch.jl    | 77 +++++++++++++++++++
 .../src/nlsolve/broyden.jl                    | 30 +++++++-
 .../src/nlsolve/lbroyden.jl                   | 29 ++++---
 5 files changed, 131 insertions(+), 21 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/linesearch.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index d518e1686..6373e9647 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,13 +1,14 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.2.1"
+version = "1.3.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
 ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -17,17 +18,18 @@ Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
-[extensions]
-SimpleNonlinearSolvePolyesterForwardDiffExt = "PolyesterForwardDiff"
-
 [weakdeps]
 PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
 
+[extensions]
+SimpleNonlinearSolvePolyesterForwardDiffExt = "PolyesterForwardDiff"
+
 [compat]
 ADTypes = "0.2.6"
 ArrayInterface = "7"
 ConcreteStructs = "0.2"
 DiffEqBase = "6.126"
+FastClosures = "0.3"
 FiniteDiff = "2"
 ForwardDiff = "0.10.3"
 LinearAlgebra = "1.9"
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 356c0b51e..c8f515cc2 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -3,14 +3,13 @@ module SimpleNonlinearSolve
 import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidations
 
 @recompile_invalidations begin
-    using ADTypes,
-        ArrayInterface, ConcreteStructs, DiffEqBase, Reexport, LinearAlgebra, SciMLBase
+    using ADTypes, ArrayInterface, ConcreteStructs, DiffEqBase, FastClosures, FiniteDiff,
+        ForwardDiff, Reexport, LinearAlgebra, SciMLBase
 
     import DiffEqBase: AbstractNonlinearTerminationMode,
         AbstractSafeNonlinearTerminationMode, AbstractSafeBestNonlinearTerminationMode,
         NonlinearSafeTerminationReturnCode, get_termination_mode,
         NONLINEARSOLVE_DEFAULT_NORM, _get_tolerance
-    using FiniteDiff, ForwardDiff
     import ForwardDiff: Dual
     import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
     import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace
@@ -26,6 +25,7 @@ abstract type AbstractNewtonAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm e
 @inline __is_extension_loaded(::Val) = false
 
 include("utils.jl")
+include("linesearch.jl")
 
 ## Nonlinear Solvers
 include("nlsolve/raphson.jl")
diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
new file mode 100644
index 000000000..fdab57bfd
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -0,0 +1,77 @@
+# This is a copy of the version in NonlinearSolve.jl. Temporarily kept here till we move
+# line searches into a dedicated package. Renamed to `__` to avoid conflicts.
+@kwdef @concrete struct __LiFukushimaLineSearch
+    lambda_0 = 1
+    beta = 1 // 2
+    sigma_1 = 1 // 1000
+    sigma_2 = 1 // 1000
+    eta = 1 // 10
+    rho = 9 // 10
+    nan_maxiters::Int = 5
+    maxiters::Int = 100
+end
+
+@concrete mutable struct __LiFukushimaLineSearchCache
+    ϕ
+    λ₀
+    β
+    σ₁
+    σ₂
+    η
+    ρ
+    α
+    nan_maxiters::Int
+    maxiters::Int
+end
+
+function (alg::__LiFukushimaLineSearch)(prob, fu, u)
+    @bb u_cache = similar(u)
+    @bb fu_cache = similar(fu)
+    T = promote_type(eltype(fu), eltype(u))
+
+    ϕ = @closure (u, δu, α) -> begin
+        @bb @. u_cache = u + α * δu
+        return norm(__eval_f(prob, fu_cache, u_cache), 2)
+    end
+
+    return __LiFukushimaLineSearchCache(ϕ, T(alg.lambda_0), T(alg.beta), T(alg.sigma_1),
+        T(alg.sigma_2), T(alg.eta), T(alg.rho), T(true), alg.nan_maxiters, alg.maxiters)
+end
+
+function (cache::__LiFukushimaLineSearchCache)(u, δu)
+    T = promote_type(eltype(u), eltype(δu))
+    ϕ = @closure α -> cache.ϕ(u, δu, α)
+
+    fx_norm = ϕ(T(0))
+
+    # Non-Blocking exit if the norm is NaN or Inf
+    !isfinite(fx_norm) && return cache.α
+
+    # Early Terminate based on Eq. 2.7
+    du_norm = norm(δu, 2)
+    fxλ_norm = ϕ(cache.α)
+    fxλ_norm ≤ cache.ρ * fx_norm - cache.σ₂ * du_norm^2 && return cache.α
+
+    λ₂, λ₁ = cache.λ₀, cache.λ₀
+    fxλp_norm = ϕ(λ₂)
+
+    if !isfinite(fxλp_norm)
+        nan_converged = false
+        for _ in 1:(cache.nan_maxiters)
+            λ₁, λ₂ = λ₂, cache.β * λ₂
+            fxλp_norm = ϕ(λ₂)
+            nan_converged = isfinite(fxλp_norm)
+            nan_converged && break
+        end
+        nan_converged || return cache.α
+    end
+
+    for i in 1:(cache.maxiters)
+        fxλp_norm = ϕ(λ₂)
+        converged = fxλp_norm ≤ (1 + cache.η) * fx_norm - cache.σ₁ * λ₂^2 * du_norm^2
+        converged && return λ₂
+        λ₁, λ₂ = λ₂, cache.β * λ₂
+    end
+
+    return cache.α
+end
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index 5f2dccdc7..82312a01d 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -1,10 +1,26 @@
 """
-    SimpleBroyden()
+    SimpleBroyden(; linesearch = Val(false))
 
 A low-overhead implementation of Broyden. This method is non-allocating on scalar
 and static array problems.
+
+If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch` [1] line search else
+no line search is used. For advanced customization of the line search, use the
+`Broyden` algorithm in `NonlinearSolve.jl`.
+
+### References
+
+[1] Li, Dong-Hui, and Masao Fukushima. "A derivative-free line search and global convergence
+of Broyden-like method for nonlinear equations." Optimization methods and software 13.3
+(2000): 181-201.
 """
-struct SimpleBroyden <: AbstractSimpleNonlinearSolveAlgorithm end
+struct SimpleBroyden{linesearch} <: AbstractSimpleNonlinearSolveAlgorithm end
+
+function SimpleBroyden(; linesearch = Val(false))
+    SimpleBroyden{SciMLBase._unwrap_val(linesearch)}()
+end
+
+__get_linesearch(::SimpleBroyden{LS}) where {LS} = Val(LS)
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
@@ -26,9 +42,16 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
 
+    ls_cache = __get_linesearch(alg) === Val(true) ?
+               __LiFukushimaLineSearch()(prob, fx, x) : nothing
+
     for _ in 1:maxiters
         @bb δx = J⁻¹ × vec(fprev)
-        @bb @. x = xo - δx
+        @bb δx .*= -1
+
+        α = ls_cache === nothing ? true : ls_cache(x, δx)
+
+        @bb @. x = xo + α * δx
         fx = __eval_f(prob, fx, x)
         @bb @. δf = fx - fprev
 
@@ -37,7 +60,6 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
         tc_sol !== nothing && return tc_sol
 
         @bb J⁻¹δf = J⁻¹ × vec(δf)
-        @bb δx .*= -1
         d = dot(δx, J⁻¹δf)
         @bb xᵀJ⁻¹ = transpose(J⁻¹) × vec(δx)
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 3a0c11ded..d949fcc8b 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -1,6 +1,6 @@
 """
-    SimpleLimitedMemoryBroyden(; threshold::Int = 27)
-    SimpleLimitedMemoryBroyden(; threshold::Val = Val(27))
+    SimpleLimitedMemoryBroyden(; threshold::Int = 27, linesearch = Val(true))
+    SimpleLimitedMemoryBroyden(; threshold::Val = Val(27), linesearch = Val(true))
 
 A limited memory implementation of Broyden. This method applies the L-BFGS scheme to
 Broyden's method. This Alogrithm unfortunately cannot non-allocating for StaticArrays
@@ -8,17 +8,26 @@ without compromising on the "simple" aspect.
 
 If the threshold is larger than the problem size, then this method will use `SimpleBroyden`.
 
-!!! warning
+If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch` [1] line search else
+no line search is used. For advanced customization of the line search, use the
+`LimitedMemoryBroyden` algorithm in `NonlinearSolve.jl`.
 
-    This method is not very stable and can diverge even for very simple problems. This has
-    mostly been tested for neural networks in DeepEquilibriumNetworks.jl.
+### References
+
+[1] Li, Dong-Hui, and Masao Fukushima. "A derivative-free line search and global convergence
+of Broyden-like method for nonlinear equations." Optimization methods and software 13.3
+(2000): 181-201.
 """
-struct SimpleLimitedMemoryBroyden{threshold} <: AbstractSimpleNonlinearSolveAlgorithm end
+struct SimpleLimitedMemoryBroyden{threshold, linesearch} <:
+       AbstractSimpleNonlinearSolveAlgorithm end
 
 __get_threshold(::SimpleLimitedMemoryBroyden{threshold}) where {threshold} = Val(threshold)
+__use_linesearch(::SimpleLimitedMemoryBroyden{Th, LS}) where {Th, LS} = Val(LS)
 
-function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27))
-    return SimpleLimitedMemoryBroyden{SciMLBase._unwrap_val(threshold)}()
+function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27),
+        linesearch = Val(true))
+    return SimpleLimitedMemoryBroyden{SciMLBase._unwrap_val(threshold),
+        SciMLBase._unwrap_val(linesearch)}()
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
@@ -45,8 +54,8 @@ end
 
     # For scalar problems / if the threshold is larger than problem size just use Broyden
     if x isa Number || length(x) ≤ η
-        return SciMLBase.__solve(prob, SimpleBroyden(), args...;
-            abstol, reltol, maxiters, termination_condition, kwargs...)
+        return SciMLBase.__solve(prob, SimpleBroyden(; linesearch = __use_linesearch(alg)),
+            args...; abstol, reltol, maxiters, termination_condition, kwargs...)
     end
 
     fx = _get_fx(prob, x)

From e822edd1287f89424bdd59a63741a3a38e5ba9d6 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 10 Jan 2024 10:24:01 -0500
Subject: [PATCH 274/375] Add Line Search to LBroyden

---
 lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl  | 1 -
 lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl | 6 +++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index 82312a01d..cc505339f 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -50,7 +50,6 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
         @bb δx .*= -1
 
         α = ls_cache === nothing ? true : ls_cache(x, δx)
-
         @bb @. x = xo + α * δx
         fx = __eval_f(prob, fx, x)
         @bb @. δf = fx - fprev
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index d949fcc8b..ccd77d38e 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -75,8 +75,12 @@ end
     Tcache = __lbroyden_threshold_cache(x, x isa StaticArray ? threshold : Val(η))
     @bb mat_cache = copy(x)
 
+    ls_cache = __get_linesearch(alg) === Val(true) ?
+               __LiFukushimaLineSearch()(prob, fx, x) : nothing
+
     for i in 1:maxiters
-        @bb @. x = xo + δx
+        α = ls_cache === nothing ? true : ls_cache(x, δx)
+        @bb @. x = xo + α * δx
         fx = __eval_f(prob, fx, x)
         @bb @. δf = fx - fo
 

From ee1b66d9f7b45eda729f20e8d64005eaae5dd3ca Mon Sep 17 00:00:00 2001
From: Vaibhav Kumar Dixit <vaibhavyashdixit@gmail.com>
Date: Wed, 10 Jan 2024 14:28:32 -0500
Subject: [PATCH 275/375] Update src/nlsolve/lbroyden.jl

---
 lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index ccd77d38e..fa3b9d3cd 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -75,7 +75,7 @@ end
     Tcache = __lbroyden_threshold_cache(x, x isa StaticArray ? threshold : Val(η))
     @bb mat_cache = copy(x)
 
-    ls_cache = __get_linesearch(alg) === Val(true) ?
+    ls_cache = __use_linesearch(alg) === Val(true) ?
                __LiFukushimaLineSearch()(prob, fx, x) : nothing
 
     for i in 1:maxiters

From 7648d58cec5b39ffc9424b49098148135a4955c6 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Fri, 12 Jan 2024 12:29:57 -0500
Subject: [PATCH 276/375] temp diable termination condition

---
 lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index cc505339f..fa0b17359 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -39,8 +39,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
     @bb δJ⁻¹n = copy(x)
     @bb δJ⁻¹ = copy(J⁻¹)
 
-    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
-        termination_condition)
+    # abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+    #     termination_condition)
 
     ls_cache = __get_linesearch(alg) === Val(true) ?
                __LiFukushimaLineSearch()(prob, fx, x) : nothing
@@ -55,8 +55,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
         @bb @. δf = fx - fprev
 
         # Termination Checks
-        tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
-        tc_sol !== nothing && return tc_sol
+        # tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+        # tc_sol !== nothing && return tc_sol
 
         @bb J⁻¹δf = J⁻¹ × vec(δf)
         d = dot(δx, J⁻¹δf)

From 20296e803f8df726172c139f260b05ebc468189a Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Fri, 12 Jan 2024 13:48:05 -0500
Subject: [PATCH 277/375] isfinite not comiling

---
 lib/SimpleNonlinearSolve/src/linesearch.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index fdab57bfd..8a93410bd 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -45,7 +45,7 @@ function (cache::__LiFukushimaLineSearchCache)(u, δu)
     fx_norm = ϕ(T(0))
 
     # Non-Blocking exit if the norm is NaN or Inf
-    !isfinite(fx_norm) && return cache.α
+    (fx_norm == Inf || fx_norm == NaN) && return cache.α
 
     # Early Terminate based on Eq. 2.7
     du_norm = norm(δu, 2)

From c222e6412ff8a186f459d6695a64036a43887bc9 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Fri, 12 Jan 2024 14:10:06 -0500
Subject: [PATCH 278/375] use DiffEqBase definitions of norm etc

---
 lib/SimpleNonlinearSolve/src/linesearch.jl | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index 8a93410bd..4d2ec4aa5 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -2,11 +2,11 @@
 # line searches into a dedicated package. Renamed to `__` to avoid conflicts.
 @kwdef @concrete struct __LiFukushimaLineSearch
     lambda_0 = 1
-    beta = 1 // 2
-    sigma_1 = 1 // 1000
-    sigma_2 = 1 // 1000
-    eta = 1 // 10
-    rho = 9 // 10
+    beta = 1.0 / 2.0
+    sigma_1 = 1.0 / 1000.0
+    sigma_2 = 1.0 / 1000.0
+    eta = 1.0 / 10.0
+    rho = 9.0 / 10.0
     nan_maxiters::Int = 5
     maxiters::Int = 100
 end
@@ -31,7 +31,7 @@ function (alg::__LiFukushimaLineSearch)(prob, fu, u)
 
     ϕ = @closure (u, δu, α) -> begin
         @bb @. u_cache = u + α * δu
-        return norm(__eval_f(prob, fu_cache, u_cache), 2)
+        return NONLINEARSOLVE_DEFAULT_NORM(__eval_f(prob, fu_cache, u_cache))
     end
 
     return __LiFukushimaLineSearchCache(ϕ, T(alg.lambda_0), T(alg.beta), T(alg.sigma_1),
@@ -42,25 +42,25 @@ function (cache::__LiFukushimaLineSearchCache)(u, δu)
     T = promote_type(eltype(u), eltype(δu))
     ϕ = @closure α -> cache.ϕ(u, δu, α)
 
-    fx_norm = ϕ(T(0))
+    fx_norm::T = ϕ(T(0))
 
     # Non-Blocking exit if the norm is NaN or Inf
-    (fx_norm == Inf || fx_norm == NaN) && return cache.α
+    DiffEqBase.NAN_CHECK(fx_norm) && return cache.α
 
     # Early Terminate based on Eq. 2.7
-    du_norm = norm(δu, 2)
+    du_norm = NONLINEARSOLVE_DEFAULT_NORM(δu)
     fxλ_norm = ϕ(cache.α)
     fxλ_norm ≤ cache.ρ * fx_norm - cache.σ₂ * du_norm^2 && return cache.α
 
     λ₂, λ₁ = cache.λ₀, cache.λ₀
     fxλp_norm = ϕ(λ₂)
 
-    if !isfinite(fxλp_norm)
+    if DiffEqBase.NAN_CHECK(fxλp_norm)
         nan_converged = false
         for _ in 1:(cache.nan_maxiters)
             λ₁, λ₂ = λ₂, cache.β * λ₂
             fxλp_norm = ϕ(λ₂)
-            nan_converged = isfinite(fxλp_norm)
+            nan_converged = DiffEqBase.NAN_CHECK(fxλp_norm)
             nan_converged && break
         end
         nan_converged || return cache.α

From 705ac0612dbc5da1982882900c22ee3a5914d0f0 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Fri, 12 Jan 2024 15:24:05 -0500
Subject: [PATCH 279/375] T(phi)

---
 lib/SimpleNonlinearSolve/src/linesearch.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index 4d2ec4aa5..b93bd21de 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -42,7 +42,7 @@ function (cache::__LiFukushimaLineSearchCache)(u, δu)
     T = promote_type(eltype(u), eltype(δu))
     ϕ = @closure α -> cache.ϕ(u, δu, α)
 
-    fx_norm::T = ϕ(T(0))
+    fx_norm = ϕ(T(0))::T
 
     # Non-Blocking exit if the norm is NaN or Inf
     DiffEqBase.NAN_CHECK(fx_norm) && return cache.α

From 5a92d0f807c97c3734297cffe887830b42adf3d4 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Fri, 12 Jan 2024 16:31:05 -0500
Subject: [PATCH 280/375] Typer assertions and simplify function eval all
 around

---
 lib/SimpleNonlinearSolve/src/linesearch.jl | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index b93bd21de..82ece8381 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -25,13 +25,11 @@ end
 end
 
 function (alg::__LiFukushimaLineSearch)(prob, fu, u)
-    @bb u_cache = similar(u)
-    @bb fu_cache = similar(fu)
     T = promote_type(eltype(fu), eltype(u))
 
     ϕ = @closure (u, δu, α) -> begin
-        @bb @. u_cache = u + α * δu
-        return NONLINEARSOLVE_DEFAULT_NORM(__eval_f(prob, fu_cache, u_cache))
+        u_cache = @. u + α * δu
+        return NONLINEARSOLVE_DEFAULT_NORM(prob.f(u_cache, prob.p))
     end
 
     return __LiFukushimaLineSearchCache(ϕ, T(alg.lambda_0), T(alg.beta), T(alg.sigma_1),
@@ -45,22 +43,22 @@ function (cache::__LiFukushimaLineSearchCache)(u, δu)
     fx_norm = ϕ(T(0))::T
 
     # Non-Blocking exit if the norm is NaN or Inf
-    DiffEqBase.NAN_CHECK(fx_norm) && return cache.α
+    DiffEqBase.NAN_CHECK(fx_norm)::Bool && return cache.α
 
     # Early Terminate based on Eq. 2.7
-    du_norm = NONLINEARSOLVE_DEFAULT_NORM(δu)
-    fxλ_norm = ϕ(cache.α)
+    du_norm = NONLINEARSOLVE_DEFAULT_NORM(δu)::T
+    fxλ_norm = ϕ(cache.α)::T
     fxλ_norm ≤ cache.ρ * fx_norm - cache.σ₂ * du_norm^2 && return cache.α
 
     λ₂, λ₁ = cache.λ₀, cache.λ₀
-    fxλp_norm = ϕ(λ₂)
+    fxλp_norm = ϕ(λ₂)::T
 
-    if DiffEqBase.NAN_CHECK(fxλp_norm)
+    if DiffEqBase.NAN_CHECK(fxλp_norm)::Bool
         nan_converged = false
         for _ in 1:(cache.nan_maxiters)
             λ₁, λ₂ = λ₂, cache.β * λ₂
             fxλp_norm = ϕ(λ₂)
-            nan_converged = DiffEqBase.NAN_CHECK(fxλp_norm)
+            nan_converged = DiffEqBase.NAN_CHECK(fxλp_norm)::Bool
             nan_converged && break
         end
         nan_converged || return cache.α

From f14726deb7624c1765425ef374b093b98ba134e0 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Fri, 12 Jan 2024 16:53:29 -0500
Subject: [PATCH 281/375] Bring back bb

---
 lib/SimpleNonlinearSolve/src/linesearch.jl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index 82ece8381..1b21974b9 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -25,11 +25,13 @@ end
 end
 
 function (alg::__LiFukushimaLineSearch)(prob, fu, u)
+    @bb u_cache = similar(u)
+    @bb fu_cache = similar(fu)
     T = promote_type(eltype(fu), eltype(u))
 
     ϕ = @closure (u, δu, α) -> begin
         u_cache = @. u + α * δu
-        return NONLINEARSOLVE_DEFAULT_NORM(prob.f(u_cache, prob.p))
+        return NONLINEARSOLVE_DEFAULT_NORM(__eval_f(prob, fu_cache, u_cache))
     end
 
     return __LiFukushimaLineSearchCache(ϕ, T(alg.lambda_0), T(alg.beta), T(alg.sigma_1),

From 8a5526a796622cb5526f361da6b135a8d70b005f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 00:40:25 -0500
Subject: [PATCH 282/375] Broyden line search works inside kernels

---
 .../src/SimpleNonlinearSolve.jl               |   2 +-
 lib/SimpleNonlinearSolve/src/linesearch.jl    | 109 +++++++++++++-----
 .../src/nlsolve/broyden.jl                    |  14 +--
 .../src/nlsolve/dfsane.jl                     |   2 +-
 .../src/nlsolve/lbroyden.jl                   |  17 ++-
 5 files changed, 96 insertions(+), 48 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index c8f515cc2..c6dd6291f 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -12,7 +12,7 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
         NONLINEARSOLVE_DEFAULT_NORM, _get_tolerance
     import ForwardDiff: Dual
     import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
-    import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace
+    import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace, _unwrap_val
     import StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, MMatrix, Size
 end
 
diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index 1b21974b9..e5d2bb93c 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -1,17 +1,17 @@
 # This is a copy of the version in NonlinearSolve.jl. Temporarily kept here till we move
-# line searches into a dedicated package. Renamed to `__` to avoid conflicts.
-@kwdef @concrete struct __LiFukushimaLineSearch
+# line searches into a dedicated package.
+@kwdef @concrete struct LiFukushimaLineSearch
     lambda_0 = 1
-    beta = 1.0 / 2.0
-    sigma_1 = 1.0 / 1000.0
-    sigma_2 = 1.0 / 1000.0
-    eta = 1.0 / 10.0
-    rho = 9.0 / 10.0
-    nan_maxiters::Int = 5
+    beta = 0.5
+    sigma_1 = 0.001
+    sigma_2 = 0.001
+    eta = 0.1
+    rho = 0.1
+    nan_maxiters = missing
     maxiters::Int = 100
 end
 
-@concrete mutable struct __LiFukushimaLineSearchCache
+@concrete mutable struct LiFukushimaLineSearchCache{T <: Union{Nothing, Int}}
     ϕ
     λ₀
     β
@@ -20,11 +20,31 @@ end
     η
     ρ
     α
-    nan_maxiters::Int
+    nan_maxiters::T
     maxiters::Int
 end
 
-function (alg::__LiFukushimaLineSearch)(prob, fu, u)
+@concrete struct StaticLiFukushimaLineSearchCache
+    f
+    p
+    λ₀
+    β
+    σ₁
+    σ₂
+    η
+    ρ
+    maxiters::Int
+end
+
+(alg::LiFukushimaLineSearch)(prob, fu, u) = __generic_init(alg, prob, fu, u)
+function (alg::LiFukushimaLineSearch)(prob, fu::SArray, u::SArray)
+    (alg.nan_maxiters === missing || alg.nan_maxiters === nothing) &&
+        return __static_init(alg, prob, fu, u)
+    @warn "`LiFukushimaLineSearch` with NaN checking is not non-allocating" maxlog=1
+    return __generic_init(alg, prob, fu, u)
+end
+
+function __generic_init(alg::LiFukushimaLineSearch, prob, fu, u)
     @bb u_cache = similar(u)
     @bb fu_cache = similar(fu)
     T = promote_type(eltype(fu), eltype(u))
@@ -34,36 +54,45 @@ function (alg::__LiFukushimaLineSearch)(prob, fu, u)
         return NONLINEARSOLVE_DEFAULT_NORM(__eval_f(prob, fu_cache, u_cache))
     end
 
-    return __LiFukushimaLineSearchCache(ϕ, T(alg.lambda_0), T(alg.beta), T(alg.sigma_1),
-        T(alg.sigma_2), T(alg.eta), T(alg.rho), T(true), alg.nan_maxiters, alg.maxiters)
+    nan_maxiters = ifelse(alg.nan_maxiters === missing, 5, alg.nan_maxiters)
+
+    return LiFukushimaLineSearchCache(ϕ, T(alg.lambda_0), T(alg.beta), T(alg.sigma_1),
+        T(alg.sigma_2), T(alg.eta), T(alg.rho), T(true), nan_maxiters, alg.maxiters)
 end
 
-function (cache::__LiFukushimaLineSearchCache)(u, δu)
+function __static_init(alg::LiFukushimaLineSearch, prob, fu, u)
+    T = promote_type(eltype(fu), eltype(u))
+    return StaticLiFukushimaLineSearchCache(prob.f, prob.p, T(alg.lambda_0), T(alg.beta),
+        T(alg.sigma_1), T(alg.sigma_2), T(alg.eta), T(alg.rho), alg.maxiters)
+end
+
+function (cache::LiFukushimaLineSearchCache)(u, δu)
     T = promote_type(eltype(u), eltype(δu))
     ϕ = @closure α -> cache.ϕ(u, δu, α)
-
-    fx_norm = ϕ(T(0))::T
+    fx_norm = ϕ(T(0))
 
     # Non-Blocking exit if the norm is NaN or Inf
-    DiffEqBase.NAN_CHECK(fx_norm)::Bool && return cache.α
+    DiffEqBase.NAN_CHECK(fx_norm) && return cache.α
 
     # Early Terminate based on Eq. 2.7
-    du_norm = NONLINEARSOLVE_DEFAULT_NORM(δu)::T
-    fxλ_norm = ϕ(cache.α)::T
+    du_norm = NONLINEARSOLVE_DEFAULT_NORM(δu)
+    fxλ_norm = ϕ(cache.α)
     fxλ_norm ≤ cache.ρ * fx_norm - cache.σ₂ * du_norm^2 && return cache.α
 
     λ₂, λ₁ = cache.λ₀, cache.λ₀
-    fxλp_norm = ϕ(λ₂)::T
-
-    if DiffEqBase.NAN_CHECK(fxλp_norm)::Bool
-        nan_converged = false
-        for _ in 1:(cache.nan_maxiters)
-            λ₁, λ₂ = λ₂, cache.β * λ₂
-            fxλp_norm = ϕ(λ₂)
-            nan_converged = DiffEqBase.NAN_CHECK(fxλp_norm)::Bool
-            nan_converged && break
+    fxλp_norm = ϕ(λ₂)
+
+    if cache.nan_maxiters !== nothing
+        if DiffEqBase.NAN_CHECK(fxλp_norm)
+            nan_converged = false
+            for _ in 1:(cache.nan_maxiters)
+                λ₁, λ₂ = λ₂, cache.β * λ₂
+                fxλp_norm = ϕ(λ₂)
+                nan_converged = DiffEqBase.NAN_CHECK(fxλp_norm)::Bool
+                nan_converged && break
+            end
+            nan_converged || return cache.α
         end
-        nan_converged || return cache.α
     end
 
     for i in 1:(cache.maxiters)
@@ -75,3 +104,25 @@ function (cache::__LiFukushimaLineSearchCache)(u, δu)
 
     return cache.α
 end
+
+function (cache::StaticLiFukushimaLineSearchCache)(u, δu)
+    T = promote_type(eltype(u), eltype(δu))
+
+    # Early Terminate based on Eq. 2.7
+    fx_norm = NONLINEARSOLVE_DEFAULT_NORM(cache.f(u, cache.p))
+    du_norm = NONLINEARSOLVE_DEFAULT_NORM(δu)
+    fxλ_norm = NONLINEARSOLVE_DEFAULT_NORM(cache.f(u .+ cache.λ₀ .* δu, cache.p))
+    fxλ_norm ≤ cache.ρ * fx_norm - cache.σ₂ * du_norm^2 && return T(true)
+
+    λ₂, λ₁ = cache.λ₀, cache.λ₀
+    fxλp_norm = NONLINEARSOLVE_DEFAULT_NORM(cache.f(u .+ λ₂ .* δu, cache.p))
+
+    for i in 1:(cache.maxiters)
+        fxλp_norm = NONLINEARSOLVE_DEFAULT_NORM(cache.f(u .+ λ₂ .* δu, cache.p))
+        converged = fxλp_norm ≤ (1 + cache.η) * fx_norm - cache.σ₁ * λ₂^2 * du_norm^2
+        converged && return λ₂
+        λ₁, λ₂ = λ₂, cache.β * λ₂
+    end
+
+    return T(true)
+end
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index fa0b17359..8eebd9518 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -16,9 +16,7 @@ of Broyden-like method for nonlinear equations." Optimization methods and softwa
 """
 struct SimpleBroyden{linesearch} <: AbstractSimpleNonlinearSolveAlgorithm end
 
-function SimpleBroyden(; linesearch = Val(false))
-    SimpleBroyden{SciMLBase._unwrap_val(linesearch)}()
-end
+SimpleBroyden(; linesearch = Val(false)) = SimpleBroyden{_unwrap_val(linesearch)}()
 
 __get_linesearch(::SimpleBroyden{LS}) where {LS} = Val(LS)
 
@@ -39,11 +37,11 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
     @bb δJ⁻¹n = copy(x)
     @bb δJ⁻¹ = copy(J⁻¹)
 
-    # abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
-    #     termination_condition)
+    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+        termination_condition)
 
     ls_cache = __get_linesearch(alg) === Val(true) ?
-               __LiFukushimaLineSearch()(prob, fx, x) : nothing
+               LiFukushimaLineSearch()(prob, fx, x) : nothing
 
     for _ in 1:maxiters
         @bb δx = J⁻¹ × vec(fprev)
@@ -55,8 +53,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
         @bb @. δf = fx - fprev
 
         # Termination Checks
-        # tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
-        # tc_sol !== nothing && return tc_sol
+        tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+        tc_sol !== nothing && return tc_sol
 
         @bb J⁻¹δf = J⁻¹ × vec(δf)
         d = dot(δx, J⁻¹δf)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index c6111b38c..9232c5445 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -56,7 +56,7 @@ end
 function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
         M::Union{Int, Val} = Val(10), γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
         nexp::Int = 2, η_strategy::F = (f_1, k, x, F) -> f_1 ./ k^2) where {F}
-    return SimpleDFSane{SciMLBase._unwrap_val(M)}(σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp,
+    return SimpleDFSane{_unwrap_val(M)}(σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp,
         η_strategy)
 end
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index fa3b9d3cd..5f144a3b6 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -1,6 +1,6 @@
 """
-    SimpleLimitedMemoryBroyden(; threshold::Int = 27, linesearch = Val(true))
-    SimpleLimitedMemoryBroyden(; threshold::Val = Val(27), linesearch = Val(true))
+    SimpleLimitedMemoryBroyden(; threshold::Int = 27, linesearch = Val(false))
+    SimpleLimitedMemoryBroyden(; threshold::Val = Val(27), linesearch = Val(false))
 
 A limited memory implementation of Broyden. This method applies the L-BFGS scheme to
 Broyden's method. This Alogrithm unfortunately cannot non-allocating for StaticArrays
@@ -25,9 +25,8 @@ __get_threshold(::SimpleLimitedMemoryBroyden{threshold}) where {threshold} = Val
 __use_linesearch(::SimpleLimitedMemoryBroyden{Th, LS}) where {Th, LS} = Val(LS)
 
 function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27),
-        linesearch = Val(true))
-    return SimpleLimitedMemoryBroyden{SciMLBase._unwrap_val(threshold),
-        SciMLBase._unwrap_val(linesearch)}()
+        linesearch = Val(false))
+    return SimpleLimitedMemoryBroyden{_unwrap_val(threshold), _unwrap_val(linesearch)}()
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
@@ -50,7 +49,7 @@ end
         termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
     threshold = __get_threshold(alg)
-    η = min(SciMLBase._unwrap_val(threshold), maxiters)
+    η = min(_unwrap_val(threshold), maxiters)
 
     # For scalar problems / if the threshold is larger than problem size just use Broyden
     if x isa Number || length(x) ≤ η
@@ -134,7 +133,7 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
 
     xo, fo, δx = res.x, res.fx, res.δx
 
-    for i in 1:(maxiters - SciMLBase._unwrap_val(threshold))
+    for i in 1:(maxiters - _unwrap_val(threshold))
         x = xo .+ δx
         fx = prob.f(x, prob.p)
         δf = fx - fo
@@ -148,8 +147,8 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
         d = dot(vᵀ, δf)
         δx = @. (δx - mvec) / d
 
-        U = Base.setindex(U, vec(δx), mod1(i, SciMLBase._unwrap_val(threshold)))
-        Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), mod1(i, SciMLBase._unwrap_val(threshold)))
+        U = Base.setindex(U, vec(δx), mod1(i, _unwrap_val(threshold)))
+        Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), mod1(i, _unwrap_val(threshold)))
 
         δx = -_restructure(fx, _matvec!!(U, Vᵀ, vec(fx)))
 

From cf2551b57d1f8d9b0c70ef60ea8b41a08a3ac4e4 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 00:42:07 -0500
Subject: [PATCH 283/375] Add linesearch to the cuda tests

---
 lib/SimpleNonlinearSolve/test/basictests.jl | 4 ++--
 lib/SimpleNonlinearSolve/test/cuda.jl       | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 7da944e7b..b0b356daf 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -105,7 +105,7 @@ end
 # --- SimpleBroyden / SimpleKlement / SimpleLimitedMemoryBroyden tests ---
 
 @testset "$(_nameof(alg))" for alg in [SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
-    SimpleLimitedMemoryBroyden()]
+    SimpleLimitedMemoryBroyden(), SimpleBroyden(; linesearch = Val(true))]
     function benchmark_nlsolve_oop(f, u0, p = 2.0)
         prob = NonlinearProblem{false}(f, u0, p)
         return solve(prob, alg, abstol = 1e-9)
@@ -164,7 +164,7 @@ end
 ## SimpleDFSane needs to allocate a history vector
 @testset "Allocation Checks: $(_nameof(alg))" for alg in (SimpleNewtonRaphson(),
     SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
-    SimpleTrustRegion(), SimpleDFSane())
+    SimpleTrustRegion(), SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)))
     @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
 
     nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
diff --git a/lib/SimpleNonlinearSolve/test/cuda.jl b/lib/SimpleNonlinearSolve/test/cuda.jl
index 9b984bc70..19e67a081 100644
--- a/lib/SimpleNonlinearSolve/test/cuda.jl
+++ b/lib/SimpleNonlinearSolve/test/cuda.jl
@@ -7,7 +7,8 @@ f!(du, u, p) = du .= u .* u .- 2
 
 @testset "Solving on GPUs" begin
     for alg in (SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(), SimpleBroyden(),
-        SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley())
+        SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
+        SimpleBroyden(; linesearch = Val(true)))
         @info "Testing $alg on CUDA"
 
         # Static Arrays
@@ -43,7 +44,8 @@ end
     prob = NonlinearProblem{false}(f, @SVector[1.0f0, 1.0f0])
 
     for alg in (SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(), SimpleBroyden(),
-        SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley())
+        SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
+        SimpleBroyden(; linesearch = Val(true)))
         @test begin
             try
                 @cuda kernel_function(prob, alg)

From 1d5eeb817528ca103514a722fd7d26231ba2506f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 00:55:36 -0500
Subject: [PATCH 284/375] Add linesearch to LBroyden

---
 .../src/nlsolve/broyden.jl                    |  2 +-
 .../src/nlsolve/lbroyden.jl                   | 20 +++++++++++--------
 lib/SimpleNonlinearSolve/test/basictests.jl   |  6 ++++--
 lib/SimpleNonlinearSolve/test/cuda.jl         |  6 ++++--
 4 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index 8eebd9518..327f2e2fb 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -6,7 +6,7 @@ and static array problems.
 
 If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch` [1] line search else
 no line search is used. For advanced customization of the line search, use the
-`Broyden` algorithm in `NonlinearSolve.jl`.
+[`Broyden`](@ref) algorithm in `NonlinearSolve.jl`.
 
 ### References
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 5f144a3b6..5f5d481c2 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -3,14 +3,13 @@
     SimpleLimitedMemoryBroyden(; threshold::Val = Val(27), linesearch = Val(false))
 
 A limited memory implementation of Broyden. This method applies the L-BFGS scheme to
-Broyden's method. This Alogrithm unfortunately cannot non-allocating for StaticArrays
-without compromising on the "simple" aspect.
+Broyden's method.
 
 If the threshold is larger than the problem size, then this method will use `SimpleBroyden`.
 
 If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch` [1] line search else
 no line search is used. For advanced customization of the line search, use the
-`LimitedMemoryBroyden` algorithm in `NonlinearSolve.jl`.
+[`LimitedMemoryBroyden`](@ref) algorithm in `NonlinearSolve.jl`.
 
 ### References
 
@@ -75,7 +74,7 @@ end
     @bb mat_cache = copy(x)
 
     ls_cache = __use_linesearch(alg) === Val(true) ?
-               __LiFukushimaLineSearch()(prob, fx, x) : nothing
+               LiFukushimaLineSearch()(prob, fx, x) : nothing
 
     for i in 1:maxiters
         α = ls_cache === nothing ? true : ls_cache(x, δx)
@@ -125,8 +124,11 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
 
     xo, δx, fo, δf = x, -fx, fx, fx
 
+    ls_cache = __use_linesearch(alg) === Val(true) ?
+               LiFukushimaLineSearch()(prob, fx, x) : nothing
+
     converged, res = __unrolled_lbroyden_initial_iterations(prob, xo, fo, δx, abstol, U, Vᵀ,
-        threshold)
+        threshold, ls_cache)
 
     converged &&
         return build_solution(prob, alg, res.x, res.fx; retcode = ReturnCode.Success)
@@ -134,7 +136,8 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
     xo, fo, δx = res.x, res.fx, res.δx
 
     for i in 1:(maxiters - _unwrap_val(threshold))
-        x = xo .+ δx
+        α = ls_cache === nothing ? true : ls_cache(xo, δx)
+        x = xo .+ α .* δx
         fx = prob.f(x, prob.p)
         δf = fx - fo
 
@@ -160,13 +163,14 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
 end
 
 @generated function __unrolled_lbroyden_initial_iterations(prob, xo, fo, δx, abstol, U,
-        Vᵀ, ::Val{threshold}) where {threshold}
+        Vᵀ, ::Val{threshold}, ls_cache) where {threshold}
     calls = []
     for i in 1:threshold
         static_idx, static_idx_p1 = Val(i - 1), Val(i)
         push!(calls,
             quote
-                x = xo .+ δx
+                α = ls_cache === nothing ? true : ls_cache(xo, δx)
+                x = xo .+ α .* δx
                 fx = prob.f(x, prob.p)
                 δf = fx - fo
 
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index b0b356daf..2c4cd6be5 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -105,7 +105,8 @@ end
 # --- SimpleBroyden / SimpleKlement / SimpleLimitedMemoryBroyden tests ---
 
 @testset "$(_nameof(alg))" for alg in [SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
-    SimpleLimitedMemoryBroyden(), SimpleBroyden(; linesearch = Val(true))]
+    SimpleLimitedMemoryBroyden(), SimpleBroyden(; linesearch = Val(true)),
+    SimpleLimitedMemoryBroyden(; linesearch = Val(true))]
     function benchmark_nlsolve_oop(f, u0, p = 2.0)
         prob = NonlinearProblem{false}(f, u0, p)
         return solve(prob, alg, abstol = 1e-9)
@@ -164,7 +165,8 @@ end
 ## SimpleDFSane needs to allocate a history vector
 @testset "Allocation Checks: $(_nameof(alg))" for alg in (SimpleNewtonRaphson(),
     SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
-    SimpleTrustRegion(), SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)))
+    SimpleTrustRegion(), SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)),
+    SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
     @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
 
     nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
diff --git a/lib/SimpleNonlinearSolve/test/cuda.jl b/lib/SimpleNonlinearSolve/test/cuda.jl
index 19e67a081..95ee60fb6 100644
--- a/lib/SimpleNonlinearSolve/test/cuda.jl
+++ b/lib/SimpleNonlinearSolve/test/cuda.jl
@@ -8,7 +8,8 @@ f!(du, u, p) = du .= u .* u .- 2
 @testset "Solving on GPUs" begin
     for alg in (SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(), SimpleBroyden(),
         SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
-        SimpleBroyden(; linesearch = Val(true)))
+        SimpleBroyden(; linesearch = Val(true)),
+        SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
         @info "Testing $alg on CUDA"
 
         # Static Arrays
@@ -45,7 +46,8 @@ end
 
     for alg in (SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(), SimpleBroyden(),
         SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
-        SimpleBroyden(; linesearch = Val(true)))
+        SimpleBroyden(; linesearch = Val(true)),
+        SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
         @test begin
             try
                 @cuda kernel_function(prob, alg)

From 223ca68a30f292910b27adde52a303c57aee73ae Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 01:04:00 -0500
Subject: [PATCH 285/375] Remove the old flaky benchmarktools allocations tests

---
 lib/SimpleNonlinearSolve/test/Project.toml  |  1 -
 lib/SimpleNonlinearSolve/test/basictests.jl | 21 +--------------------
 2 files changed, 1 insertion(+), 21 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
index 4442a15a6..cbf18f66c 100644
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -1,6 +1,5 @@
 [deps]
 AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
-BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 2c4cd6be5..726930072 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,4 +1,4 @@
-using AllocCheck, BenchmarkTools, LinearSolve, SimpleNonlinearSolve, StaticArrays, Random,
+using AllocCheck, LinearSolve, SimpleNonlinearSolve, StaticArrays, Random,
     LinearAlgebra, Test, ForwardDiff, DiffEqBase
 import PolyesterForwardDiff
 
@@ -57,13 +57,6 @@ const TERMINATION_CONDITIONS = [
         end
     end
 
-    @testset "Allocations: Static Array and Scalars" begin
-        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, $(@SVector[1.0, 1.0]),
-            2.0; autodiff = AutoForwardDiff())) < 200
-        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, 1.0, 2.0;
-            autodiff = AutoForwardDiff())) == 0
-    end
-
     @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
         u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
@@ -89,11 +82,6 @@ end
         end
     end
 
-    @testset "Allocations: Static Array and Scalars" begin
-        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, 1.0, 2.0;
-            autodiff = AutoForwardDiff())) == 0
-    end
-
     @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
         u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
@@ -129,13 +117,6 @@ end
         @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
     end
 
-    @testset "Allocations: Static Array and Scalars" begin
-        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, $(@SVector[1.0, 1.0]),
-            2.0)) < 200
-        allocs = alg isa SimpleDFSane ? 144 : 0
-        @test (@ballocated $(benchmark_nlsolve_oop)($quadratic_f, 1.0, 2.0)) == allocs
-    end
-
     @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
         u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 

From 545e3358f8967b4aeef3e38f24251129d5b9f77f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 01:34:10 -0500
Subject: [PATCH 286/375] DFSane non-allocating for scalars now

---
 lib/SimpleNonlinearSolve/Project.toml         |  3 +++
 .../SimpleNonlinearSolveStaticArraysExt.jl    |  7 ++++++
 lib/SimpleNonlinearSolve/src/linesearch.jl    |  3 ++-
 .../src/nlsolve/dfsane.jl                     | 23 ++++++++++---------
 lib/SimpleNonlinearSolve/test/basictests.jl   |  3 +--
 5 files changed, 25 insertions(+), 14 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 6373e9647..493766ead 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -20,9 +20,11 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [weakdeps]
 PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 
 [extensions]
 SimpleNonlinearSolvePolyesterForwardDiffExt = "PolyesterForwardDiff"
+SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
 
 [compat]
 ADTypes = "0.2.6"
@@ -38,4 +40,5 @@ PrecompileTools = "1"
 Reexport = "1"
 SciMLBase = "2.7"
 StaticArraysCore = "1.4"
+StaticArrays = "1"
 julia = "1.9"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl
new file mode 100644
index 000000000..90318a82a
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl
@@ -0,0 +1,7 @@
+module SimpleNonlinearSolveStaticArraysExt
+
+using SimpleNonlinearSolve
+
+@inline SimpleNonlinearSolve.__is_extension_loaded(::Val{:StaticArrays}) = true
+
+end
diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index e5d2bb93c..13f0c28af 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -37,7 +37,8 @@ end
 end
 
 (alg::LiFukushimaLineSearch)(prob, fu, u) = __generic_init(alg, prob, fu, u)
-function (alg::LiFukushimaLineSearch)(prob, fu::SArray, u::SArray)
+function (alg::LiFukushimaLineSearch)(prob, fu::Union{Number, SArray},
+        u::Union{Number, SArray})
     (alg.nan_maxiters === missing || alg.nan_maxiters === nothing) &&
         return __static_init(alg, prob, fu, u)
     @warn "`LiFukushimaLineSearch` with NaN checking is not non-allocating" maxlog=1
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 9232c5445..6931e6101 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -9,21 +9,21 @@ see the paper [1].
 
 ### Keyword Arguments
 
-  - `σ_min`: the minimum value of the spectral coefficient `σ_k` which is related to the step
-    size in the algorithm. Defaults to `1e-10`.
-  - `σ_max`: the maximum value of the spectral coefficient `σ_k` which is related to the step
-    size in the algorithm. Defaults to `1e10`.
+  - `σ_min`: the minimum value of the spectral coefficient `σ_k` which is related to the
+    step size in the algorithm. Defaults to `1e-10`.
+  - `σ_max`: the maximum value of the spectral coefficient `σ_k` which is related to the
+    step size in the algorithm. Defaults to `1e10`.
   - `σ_1`: the initial value of the spectral coefficient `σ_k` which is related to the step
     size in the algorithm.. Defaults to `1.0`.
   - `M`: The monotonicity of the algorithm is determined by a this positive integer.
     A value of 1 for `M` would result in strict monotonicity in the decrease of the L2-norm
-    of the function `f`. However, higher values allow for more flexibility in this reduction.
-    Despite this, the algorithm still ensures global convergence through the use of a
-    non-monotone line-search algorithm that adheres to the Grippo-Lampariello-Lucidi
+    of the function `f`. However, higher values allow for more flexibility in this
+    reduction. Despite this, the algorithm still ensures global convergence through the use
+    of a non-monotone line-search algorithm that adheres to the Grippo-Lampariello-Lucidi
     condition. Values in the range of 5 to 20 are usually sufficient, but some cases may call
     for a higher value of `M`. The default setting is 10.
-  - `γ`: a parameter that influences if a proposed step will be accepted. Higher value of `γ`
-    will make the algorithm more restrictive in accepting steps. Defaults to `1e-4`.
+  - `γ`: a parameter that influences if a proposed step will be accepted. Higher value of
+    `γ` will make the algorithm more restrictive in accepting steps. Defaults to `1e-4`.
   - `τ_min`: if a step is rejected the new step size will get multiplied by factor, and this
     parameter is the minimum value of that factor. Defaults to `0.1`.
   - `τ_max`: if a step is rejected the new step size will get multiplied by factor, and this
@@ -31,7 +31,7 @@ see the paper [1].
   - `nexp`: the exponent of the loss, i.e. ``f_k=||F(x_k)||^{nexp}``. The paper uses
     `nexp ∈ {1,2}`. Defaults to `2`.
   - `η_strategy`:  function to determine the parameter `η_k`, which enables growth
-    of ``||F||^2``. Called as ``η_k = η_strategy(f_1, k, x, F)`` with `f_1` initialized as
+    of ``||F||^2``. Called as `η_k = η_strategy(f_1, k, x, F)` with `f_1` initialized as
     ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and
     `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to
     ``||F||^2 / k^2``.
@@ -83,7 +83,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...
     α_1 = one(T)
     f_1 = fx_norm
 
-    history_f_k = if x isa SArray
+    history_f_k = if x isa SArray ||
+                     (x isa Number && __is_extension_loaded(Val(:StaticArrays)))
         ones(SVector{M, T}) * fx_norm
     else
         fill(fx_norm, M)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 726930072..5938197c7 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -158,8 +158,7 @@ end
         @test true
     catch e
         @error e
-        # History Vector Allocates
-        @test false broken=(alg isa SimpleDFSane)
+        @test false
     end
 
     # ForwardDiff allocates for hessian since we don't propagate the chunksize

From 2ff6536cbd928a0757348ccc8b427b256808ecdb Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 02:53:14 -0500
Subject: [PATCH 287/375] Resolve tolerances differently for kernel launches

---
 .../src/SimpleNonlinearSolve.jl                    | 11 ++++++++++-
 lib/SimpleNonlinearSolve/src/ad.jl                 | 14 ++++++++++++++
 .../src/bracketing/bisection.jl                    |  2 +-
 lib/SimpleNonlinearSolve/src/bracketing/brent.jl   |  2 +-
 lib/SimpleNonlinearSolve/src/bracketing/falsi.jl   |  2 +-
 lib/SimpleNonlinearSolve/src/bracketing/itp.jl     |  2 +-
 lib/SimpleNonlinearSolve/src/bracketing/ridder.jl  |  2 +-
 lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl   |  8 +++++++-
 lib/SimpleNonlinearSolve/src/utils.jl              |  7 +++++++
 lib/SimpleNonlinearSolve/test/cuda.jl              |  2 +-
 10 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index c6dd6291f..8a4f2fe27 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -9,7 +9,7 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
     import DiffEqBase: AbstractNonlinearTerminationMode,
         AbstractSafeNonlinearTerminationMode, AbstractSafeBestNonlinearTerminationMode,
         NonlinearSafeTerminationReturnCode, get_termination_mode,
-        NONLINEARSOLVE_DEFAULT_NORM, _get_tolerance
+        NONLINEARSOLVE_DEFAULT_NORM
     import ForwardDiff: Dual
     import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
     import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace, _unwrap_val
@@ -65,6 +65,15 @@ function SciMLBase.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSol
     return SciMLBase.__solve(prob, alg, args...; kwargs...)
 end
 
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{<:Number, <:SArray}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm, args...; abstol = nothing,
+        reltol = nothing, kwargs...)
+    _abstol = __get_tolerance(prob.u0, abstol, eltype(prob.u0))
+    _reltol = __get_tolerance(prob.u0, reltol, eltype(prob.u0))
+    return SciMLBase.__solve(prob, alg, args...; abstol = _abstol, reltol = _reltol,
+        kwargs...)
+end
+
 @setup_workload begin
     for T in (Float32, Float64)
         prob_no_brack_scalar = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 574904bcc..11aa95476 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -8,6 +8,20 @@ function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, <:AbstractArray}
 end
 
 # Handle Ambiguities
+for algType in (SimpleNewtonRaphson, SimpleDFSane, SimpleTrustRegion, SimpleBroyden,
+    SimpleLimitedMemoryBroyden, SimpleKlement, SimpleHalley)
+    @eval begin
+        function SciMLBase.solve(prob::NonlinearProblem{uType, iip,
+                    <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
+                alg::$(algType), args...; kwargs...) where {uType, T, V, P, iip}
+            sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
+            dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
+            return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
+                sol.stats, sol.original)
+        end
+    end
+end
+
 for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
     @eval begin
         function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
index 66418b3e0..38f9cb9eb 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
@@ -26,7 +26,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
-    abstol = _get_tolerance(abstol,
+    abstol = __get_tolerance(nothing, abstol,
         promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
index 75497f379..f37c45e4a 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
@@ -13,7 +13,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
     fl, fr = f(left), f(right)
     ϵ = eps(convert(typeof(fl), 1))
 
-    abstol = _get_tolerance(abstol,
+    abstol = __get_tolerance(nothing, abstol,
         promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
index 9db7d6cf1..86086f81a 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
@@ -12,7 +12,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
-    abstol = _get_tolerance(abstol,
+    abstol = __get_tolerance(nothing, abstol,
         promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index fd46de6c3..cce5eafea 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -58,7 +58,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
-    abstol = _get_tolerance(abstol,
+    abstol = __get_tolerance(nothing, abstol,
         promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
index 20e0db489..cd18060d5 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
@@ -12,7 +12,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
-    abstol = _get_tolerance(abstol,
+    abstol = __get_tolerance(nothing, abstol,
         promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 5f5d481c2..c04fa20ee 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -28,6 +28,12 @@ function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27),
     return SimpleLimitedMemoryBroyden{_unwrap_val(threshold), _unwrap_val(linesearch)}()
 end
 
+function SciMLBase.solve(prob::NonlinearProblem{<:Union{<:Number, <:SArray}},
+        alg::SimpleLimitedMemoryBroyden, args...; kwargs...)
+    # Don't resolve the `abstol` and `reltol` here
+    return SciMLBase.__solve(prob, alg, args...; kwargs...)
+end
+
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
         args...; termination_condition = nothing, kwargs...)
     if prob.u0 isa SArray
@@ -120,7 +126,7 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
 
     U, Vᵀ = __init_low_rank_jacobian(vec(x), vec(fx), threshold)
 
-    abstol = DiffEqBase._get_tolerance(abstol, eltype(x))
+    abstol = __get_tolerance(x, abstol, eltype(x))
 
     xo, δx, fo, δf = x, -fx, fx, fx
 
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index c6b9f2004..de23e9c58 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -370,3 +370,10 @@ end
 
 @inline __reshape(x::Number, args...) = x
 @inline __reshape(x::AbstractArray, args...) = reshape(x, args...)
+
+# Override cases which might be used in a kernel launch
+__get_tolerance(x, η, ::Type{T}) where {T} = DiffEqBase._get_tolerance(η, T)
+function __get_tolerance(x::Union{SArray, Number}, ::Nothing, ::Type{T}) where {T}
+    η = real(oneunit(T)) * (eps(real(one(T))))^(real(T)(0.8))
+    return T(η)
+end
diff --git a/lib/SimpleNonlinearSolve/test/cuda.jl b/lib/SimpleNonlinearSolve/test/cuda.jl
index 95ee60fb6..fddf751de 100644
--- a/lib/SimpleNonlinearSolve/test/cuda.jl
+++ b/lib/SimpleNonlinearSolve/test/cuda.jl
@@ -37,7 +37,7 @@ f!(du, u, p) = du .= u .* u .- 2
 end
 
 function kernel_function(prob, alg)
-    solve(prob, alg; abstol = 1.0f-6, reltol = 1.0f-6)
+    solve(prob, alg)
     return nothing
 end
 

From a1ce1cfc6048c7c080de17345e6cc7b458c68dd8 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 04:55:41 -0500
Subject: [PATCH 288/375] Add alpha scaling

---
 lib/SimpleNonlinearSolve/src/linesearch.jl    |  3 +-
 .../src/nlsolve/broyden.jl                    | 33 +++++++---
 .../src/nlsolve/lbroyden.jl                   | 60 ++++++++++++-------
 lib/SimpleNonlinearSolve/src/utils.jl         | 10 ++--
 4 files changed, 69 insertions(+), 37 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index 13f0c28af..fadf69cdc 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -112,11 +112,10 @@ function (cache::StaticLiFukushimaLineSearchCache)(u, δu)
     # Early Terminate based on Eq. 2.7
     fx_norm = NONLINEARSOLVE_DEFAULT_NORM(cache.f(u, cache.p))
     du_norm = NONLINEARSOLVE_DEFAULT_NORM(δu)
-    fxλ_norm = NONLINEARSOLVE_DEFAULT_NORM(cache.f(u .+ cache.λ₀ .* δu, cache.p))
+    fxλ_norm = NONLINEARSOLVE_DEFAULT_NORM(cache.f(u .+ δu, cache.p))
     fxλ_norm ≤ cache.ρ * fx_norm - cache.σ₂ * du_norm^2 && return T(true)
 
     λ₂, λ₁ = cache.λ₀, cache.λ₀
-    fxλp_norm = NONLINEARSOLVE_DEFAULT_NORM(cache.f(u .+ λ₂ .* δu, cache.p))
 
     for i in 1:(cache.maxiters)
         fxλp_norm = NONLINEARSOLVE_DEFAULT_NORM(cache.f(u .+ λ₂ .* δu, cache.p))
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index 327f2e2fb..d670c0556 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -1,12 +1,16 @@
 """
-    SimpleBroyden(; linesearch = Val(false))
+    SimpleBroyden(; linesearch = Val(false), alpha = nothing)
 
 A low-overhead implementation of Broyden. This method is non-allocating on scalar
 and static array problems.
 
-If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch` [1] line search else
-no line search is used. For advanced customization of the line search, use the
-[`Broyden`](@ref) algorithm in `NonlinearSolve.jl`.
+### Keyword Arguments
+
+  * `linesearch`: If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch`
+    [1] line search else no line search is used. For advanced customization of the line
+    search, use the [`Broyden`](@ref) algorithm in `NonlinearSolve.jl`.
+  * `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we
+    will compute the scaling using `2 * norm(fu) / max(norm(u), true)`.
 
 ### References
 
@@ -14,9 +18,13 @@ no line search is used. For advanced customization of the line search, use the
 of Broyden-like method for nonlinear equations." Optimization methods and software 13.3
 (2000): 181-201.
 """
-struct SimpleBroyden{linesearch} <: AbstractSimpleNonlinearSolveAlgorithm end
+@concrete struct SimpleBroyden{linesearch} <: AbstractSimpleNonlinearSolveAlgorithm
+    alpha
+end
 
-SimpleBroyden(; linesearch = Val(false)) = SimpleBroyden{_unwrap_val(linesearch)}()
+function SimpleBroyden(; linesearch = Val(false), alpha = nothing)
+    return SimpleBroyden{_unwrap_val(linesearch)}(alpha)
+end
 
 __get_linesearch(::SimpleBroyden{LS}) where {LS} = Val(LS)
 
@@ -25,13 +33,22 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
         termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
     fx = _get_fx(prob, x)
+    T = promote_type(eltype(x), eltype(fx))
 
     @bb xo = copy(x)
     @bb δx = copy(x)
     @bb δf = copy(fx)
     @bb fprev = copy(fx)
 
-    J⁻¹ = __init_identity_jacobian(fx, x)
+    if alg.alpha === nothing
+        fx_norm = NONLINEARSOLVE_DEFAULT_NORM(fx)
+        x_norm = NONLINEARSOLVE_DEFAULT_NORM(x)
+        init_α = ifelse(fx_norm ≥ 1e-5, max(x_norm, T(true)) / (2 * fx_norm), T(true))
+    else
+        init_α = inv(alg.alpha)
+    end
+
+    J⁻¹ = __init_identity_jacobian(fx, x, init_α)
     @bb J⁻¹δf = copy(x)
     @bb xᵀJ⁻¹ = copy(x)
     @bb δJ⁻¹n = copy(x)
@@ -47,7 +64,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
         @bb δx = J⁻¹ × vec(fprev)
         @bb δx .*= -1
 
-        α = ls_cache === nothing ? true : ls_cache(x, δx)
+        α = ls_cache === nothing ? true : ls_cache(xo, δx)
         @bb @. x = xo + α * δx
         fx = __eval_f(prob, fx, x)
         @bb @. δf = fx - fprev
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index c04fa20ee..c35dec388 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -1,15 +1,20 @@
 """
-    SimpleLimitedMemoryBroyden(; threshold::Int = 27, linesearch = Val(false))
-    SimpleLimitedMemoryBroyden(; threshold::Val = Val(27), linesearch = Val(false))
+    SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27),
+        linesearch = Val(false), alpha = nothing)
 
 A limited memory implementation of Broyden. This method applies the L-BFGS scheme to
 Broyden's method.
 
 If the threshold is larger than the problem size, then this method will use `SimpleBroyden`.
 
-If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch` [1] line search else
-no line search is used. For advanced customization of the line search, use the
-[`LimitedMemoryBroyden`](@ref) algorithm in `NonlinearSolve.jl`.
+### Keyword Arguments:
+
+  - `linesearch`: If `linesearch` is `Val(true)`, then we use the
+    `LiFukushimaLineSearch` [1] line search else no line search is used. For advanced
+    customization of the line search, use the [`LimitedMemoryBroyden`](@ref) algorithm in
+    `NonlinearSolve.jl`.
+  - `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we
+    will compute the scaling using `2 * norm(fu) / max(norm(u), true)`.
 
 ### References
 
@@ -17,20 +22,22 @@ no line search is used. For advanced customization of the line search, use the
 of Broyden-like method for nonlinear equations." Optimization methods and software 13.3
 (2000): 181-201.
 """
-struct SimpleLimitedMemoryBroyden{threshold, linesearch} <:
-       AbstractSimpleNonlinearSolveAlgorithm end
+@concrete struct SimpleLimitedMemoryBroyden{threshold, linesearch} <:
+                 AbstractSimpleNonlinearSolveAlgorithm
+    alpha
+end
 
 __get_threshold(::SimpleLimitedMemoryBroyden{threshold}) where {threshold} = Val(threshold)
 __use_linesearch(::SimpleLimitedMemoryBroyden{Th, LS}) where {Th, LS} = Val(LS)
 
 function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27),
-        linesearch = Val(false))
-    return SimpleLimitedMemoryBroyden{_unwrap_val(threshold), _unwrap_val(linesearch)}()
+        linesearch = Val(false), alpha = nothing)
+    return SimpleLimitedMemoryBroyden{_unwrap_val(threshold), _unwrap_val(linesearch)}(alpha)
 end
 
+# Don't resolve the `abstol` and `reltol` here
 function SciMLBase.solve(prob::NonlinearProblem{<:Union{<:Number, <:SArray}},
         alg::SimpleLimitedMemoryBroyden, args...; kwargs...)
-    # Don't resolve the `abstol` and `reltol` here
     return SciMLBase.__solve(prob, alg, args...; kwargs...)
 end
 
@@ -133,8 +140,17 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
     ls_cache = __use_linesearch(alg) === Val(true) ?
                LiFukushimaLineSearch()(prob, fx, x) : nothing
 
+    T = promote_type(eltype(x), eltype(fx))
+    if alg.alpha === nothing
+        fx_norm = NONLINEARSOLVE_DEFAULT_NORM(fx)
+        x_norm = NONLINEARSOLVE_DEFAULT_NORM(x)
+        init_α = ifelse(fx_norm ≥ 1e-5, max(x_norm, T(true)) / (2 * fx_norm), T(true))
+    else
+        init_α = inv(alg.alpha)
+    end
+
     converged, res = __unrolled_lbroyden_initial_iterations(prob, xo, fo, δx, abstol, U, Vᵀ,
-        threshold, ls_cache)
+        threshold, ls_cache, init_α)
 
     converged &&
         return build_solution(prob, alg, res.x, res.fx; retcode = ReturnCode.Success)
@@ -150,8 +166,8 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
         maximum(abs, fx) ≤ abstol &&
             return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
 
-        vᵀ = _restructure(x, _rmatvec!!(U, Vᵀ, vec(δx)))
-        mvec = _restructure(x, _matvec!!(U, Vᵀ, vec(δf)))
+        vᵀ = _restructure(x, _rmatvec!!(U, Vᵀ, vec(δx), init_α))
+        mvec = _restructure(x, _matvec!!(U, Vᵀ, vec(δf), init_α))
 
         d = dot(vᵀ, δf)
         δx = @. (δx - mvec) / d
@@ -159,7 +175,7 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
         U = Base.setindex(U, vec(δx), mod1(i, _unwrap_val(threshold)))
         Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), mod1(i, _unwrap_val(threshold)))
 
-        δx = -_restructure(fx, _matvec!!(U, Vᵀ, vec(fx)))
+        δx = -_restructure(fx, _matvec!!(U, Vᵀ, vec(fx), init_α))
 
         xo = x
         fo = fx
@@ -169,7 +185,7 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
 end
 
 @generated function __unrolled_lbroyden_initial_iterations(prob, xo, fo, δx, abstol, U,
-        Vᵀ, ::Val{threshold}, ls_cache) where {threshold}
+        Vᵀ, ::Val{threshold}, ls_cache, init_α) where {threshold}
     calls = []
     for i in 1:threshold
         static_idx, static_idx_p1 = Val(i - 1), Val(i)
@@ -185,8 +201,8 @@ end
                 _U = __first_n_getindex(U, $(static_idx))
                 _Vᵀ = __first_n_getindex(Vᵀ, $(static_idx))
 
-                vᵀ = _restructure(x, _rmatvec!!(_U, _Vᵀ, vec(δx)))
-                mvec = _restructure(x, _matvec!!(_U, _Vᵀ, vec(δf)))
+                vᵀ = _restructure(x, _rmatvec!!(_U, _Vᵀ, vec(δx), init_α))
+                mvec = _restructure(x, _matvec!!(_U, _Vᵀ, vec(δf), init_α))
 
                 d = dot(vᵀ, δf)
                 δx = @. (δx - mvec) / d
@@ -196,7 +212,7 @@ end
 
                 _U = __first_n_getindex(U, $(static_idx_p1))
                 _Vᵀ = __first_n_getindex(Vᵀ, $(static_idx_p1))
-                δx = -_restructure(fx, _matvec!!(_U, _Vᵀ, vec(fx)))
+                δx = -_restructure(fx, _matvec!!(_U, _Vᵀ, vec(fx), init_α))
 
                 xo = x
                 fo = fx
@@ -226,8 +242,8 @@ function _rmatvec!!(y, xᵀU, U, Vᵀ, x)
     return y
 end
 
-@inline _rmatvec!!(::Nothing, Vᵀ, x) = -x
-@inline _rmatvec!!(U, Vᵀ, x) = __mapTdot(__mapdot(x, U), Vᵀ) .- x
+@inline _rmatvec!!(::Nothing, Vᵀ, x, init_α) = -x .* init_α
+@inline _rmatvec!!(U, Vᵀ, x, init_α) = __mapTdot(__mapdot(x, U), Vᵀ) .- x .* init_α
 
 function _matvec!!(y, Vᵀx, U, Vᵀ, x)
     # (-I + UVᵀ) × x
@@ -244,8 +260,8 @@ function _matvec!!(y, Vᵀx, U, Vᵀ, x)
     return y
 end
 
-@inline _matvec!!(::Nothing, Vᵀ, x) = -x
-@inline _matvec!!(U, Vᵀ, x) = __mapTdot(__mapdot(x, Vᵀ), U) .- x
+@inline _matvec!!(::Nothing, Vᵀ, x, init_α) = -x .* init_α
+@inline _matvec!!(U, Vᵀ, x, init_α) = __mapTdot(__mapdot(x, Vᵀ), U) .- x .* init_α
 
 function __mapdot(x::SVector{S1}, Y::SVector{S2, <:SVector{S1}}) where {S1, S2}
     return map(Base.Fix1(dot, x), Y)
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index de23e9c58..c150522da 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -214,12 +214,12 @@ function compute_jacobian_and_hessian(ad::AutoFiniteDiff, prob, fx, x)
     end
 end
 
-__init_identity_jacobian(u::Number, _) = one(u)
+__init_identity_jacobian(u::Number, fu, α = true) = oftype(u, α)
 __init_identity_jacobian!!(J::Number) = one(J)
-function __init_identity_jacobian(u, fu)
+function __init_identity_jacobian(u, fu, α = true)
     J = similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u))
     fill!(J, zero(eltype(J)))
-    J[diagind(J)] .= one(eltype(J))
+    J[diagind(J)] .= eltype(J)(α)
     return J
 end
 function __init_identity_jacobian!!(J)
@@ -231,9 +231,9 @@ function __init_identity_jacobian!!(J::AbstractVector)
     fill!(J, one(eltype(J)))
     return J
 end
-function __init_identity_jacobian(u::StaticArray, fu)
+function __init_identity_jacobian(u::StaticArray, fu, α = true)
     S1, S2 = length(fu), length(u)
-    J = SMatrix{S1, S2, eltype(u)}(I)
+    J = SMatrix{S1, S2, eltype(u)}(I * α)
     return J
 end
 function __init_identity_jacobian!!(J::SMatrix{S1, S2}) where {S1, S2}

From ba289bedc8bea004dcf785ffa1143965a58d707d Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 05:50:30 -0500
Subject: [PATCH 289/375] Resolve ambiguities

---
 .../src/SimpleNonlinearSolve.jl                   |  9 ---------
 lib/SimpleNonlinearSolve/src/ad.jl                | 15 ---------------
 lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl   |  4 ++--
 lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl  |  6 ------
 lib/SimpleNonlinearSolve/src/utils.jl             |  4 ++--
 5 files changed, 4 insertions(+), 34 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 8a4f2fe27..73eda9977 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -65,15 +65,6 @@ function SciMLBase.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSol
     return SciMLBase.__solve(prob, alg, args...; kwargs...)
 end
 
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{<:Number, <:SArray}},
-        alg::AbstractSimpleNonlinearSolveAlgorithm, args...; abstol = nothing,
-        reltol = nothing, kwargs...)
-    _abstol = __get_tolerance(prob.u0, abstol, eltype(prob.u0))
-    _reltol = __get_tolerance(prob.u0, reltol, eltype(prob.u0))
-    return SciMLBase.__solve(prob, alg, args...; abstol = _abstol, reltol = _reltol,
-        kwargs...)
-end
-
 @setup_workload begin
     for T in (Float32, Float64)
         prob_no_brack_scalar = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 11aa95476..a4a777be6 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -7,21 +7,6 @@ function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, <:AbstractArray}
         sol.original)
 end
 
-# Handle Ambiguities
-for algType in (SimpleNewtonRaphson, SimpleDFSane, SimpleTrustRegion, SimpleBroyden,
-    SimpleLimitedMemoryBroyden, SimpleKlement, SimpleHalley)
-    @eval begin
-        function SciMLBase.solve(prob::NonlinearProblem{uType, iip,
-                    <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
-                alg::$(algType), args...; kwargs...) where {uType, T, V, P, iip}
-            sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
-            dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
-            return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
-                sol.stats, sol.original)
-        end
-    end
-end
-
 for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
     @eval begin
         function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index d670c0556..ad19d765b 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -6,10 +6,10 @@ and static array problems.
 
 ### Keyword Arguments
 
-  * `linesearch`: If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch`
+  - `linesearch`: If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch`
     [1] line search else no line search is used. For advanced customization of the line
     search, use the [`Broyden`](@ref) algorithm in `NonlinearSolve.jl`.
-  * `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we
+  - `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we
     will compute the scaling using `2 * norm(fu) / max(norm(u), true)`.
 
 ### References
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index c35dec388..eb7b1bbaf 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -35,12 +35,6 @@ function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27),
     return SimpleLimitedMemoryBroyden{_unwrap_val(threshold), _unwrap_val(linesearch)}(alpha)
 end
 
-# Don't resolve the `abstol` and `reltol` here
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{<:Number, <:SArray}},
-        alg::SimpleLimitedMemoryBroyden, args...; kwargs...)
-    return SciMLBase.__solve(prob, alg, args...; kwargs...)
-end
-
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
         args...; termination_condition = nothing, kwargs...)
     if prob.u0 isa SArray
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index c150522da..7390f5e3c 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -282,8 +282,8 @@ function init_termination_cache(abstol, reltol, du, u, ::Nothing)
 end
 function init_termination_cache(abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode)
     T = promote_type(eltype(du), eltype(u))
-    abstol !== nothing && (abstol = T(abstol))
-    reltol !== nothing && (reltol = T(reltol))
+    abstol = __get_tolerance(u, abstol, T)
+    reltol = __get_tolerance(u, reltol, T)
     tc_cache = init(du, u, tc; abstol, reltol)
     return DiffEqBase.get_abstol(tc_cache), DiffEqBase.get_reltol(tc_cache), tc_cache
 end

From 4d35fc65b31b362a3114828ddc1bfaf5da0fb7cc Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 06:04:49 -0500
Subject: [PATCH 290/375] More of the 23 tests are working

---
 lib/SimpleNonlinearSolve/test/23_test_problems.jl | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
index bc82145a9..18ac5e9bd 100644
--- a/lib/SimpleNonlinearSolve/test/23_test_problems.jl
+++ b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
@@ -70,12 +70,9 @@ end
     alg_ops = (SimpleBroyden(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 4, 5, 6, 11, 12, 13, 14]
+    broken_tests[alg_ops[1]] = [1, 5, 11]
 
-    skip_tests = Dict(alg => Int[] for alg in alg_ops)
-    skip_tests[alg_ops[1]] = [2, 22]
-
-    test_on_library(problems, dicts, alg_ops, broken_tests; skip_tests)
+    test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
 @testset "SimpleKlement 23 Test Problems" begin

From 7e2fc08021ccab73af2b716ddaecb68891f608dc Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 09:26:45 -0500
Subject: [PATCH 291/375] Update src/linesearch.jl

Co-authored-by: Vaibhav Kumar Dixit <vaibhavyashdixit@gmail.com>
---
 lib/SimpleNonlinearSolve/src/linesearch.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index fadf69cdc..235ee0bba 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -51,7 +51,7 @@ function __generic_init(alg::LiFukushimaLineSearch, prob, fu, u)
     T = promote_type(eltype(fu), eltype(u))
 
     ϕ = @closure (u, δu, α) -> begin
-        u_cache = @. u + α * δu
+        @. u_cache = u + α * δu
         return NONLINEARSOLVE_DEFAULT_NORM(__eval_f(prob, fu_cache, u_cache))
     end
 

From 34289efb5e348c1e23f5d7933712a01ef10aee58 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 14 Jan 2024 09:27:34 -0500
Subject: [PATCH 292/375] Update src/linesearch.jl

---
 lib/SimpleNonlinearSolve/src/linesearch.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index 235ee0bba..c33253f63 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -51,7 +51,7 @@ function __generic_init(alg::LiFukushimaLineSearch, prob, fu, u)
     T = promote_type(eltype(fu), eltype(u))
 
     ϕ = @closure (u, δu, α) -> begin
-        @. u_cache = u + α * δu
+        @bb @. u_cache = u + α * δu
         return NONLINEARSOLVE_DEFAULT_NORM(__eval_f(prob, fu_cache, u_cache))
     end
 

From 36c2c838e329cf13d957778ac0797caab938d711 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 15 Jan 2024 02:58:55 -0500
Subject: [PATCH 293/375] Clean up some of the docs

---
 lib/SimpleNonlinearSolve/Project.toml            |  2 +-
 lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl  | 14 ++++----------
 lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl   |  8 +-------
 lib/SimpleNonlinearSolve/src/nlsolve/klement.jl  |  2 +-
 lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl | 12 +++---------
 5 files changed, 10 insertions(+), 28 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 493766ead..ca40b1ed6 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.3.0"
+version = "1.3.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index ad19d765b..15e544795 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -1,22 +1,16 @@
 """
     SimpleBroyden(; linesearch = Val(false), alpha = nothing)
 
-A low-overhead implementation of Broyden. This method is non-allocating on scalar
-and static array problems.
+A low-overhead implementation of Broyden. This method is non-allocating on scalar and static
+array problems.
 
 ### Keyword Arguments
 
   - `linesearch`: If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch`
-    [1] line search else no line search is used. For advanced customization of the line
-    search, use the [`Broyden`](@ref) algorithm in `NonlinearSolve.jl`.
+    [li2000derivative](@cite) line search else no line search is used. For advanced
+    customization of the line search, use `Broyden` from `NonlinearSolve.jl`.
   - `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we
     will compute the scaling using `2 * norm(fu) / max(norm(u), true)`.
-
-### References
-
-[1] Li, Dong-Hui, and Masao Fukushima. "A derivative-free line search and global convergence
-of Broyden-like method for nonlinear equations." Optimization methods and software 13.3
-(2000): 181-201.
 """
 @concrete struct SimpleBroyden{linesearch} <: AbstractSimpleNonlinearSolveAlgorithm
     alpha
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 6931e6101..856e31fd4 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -5,7 +5,7 @@
 
 A low-overhead implementation of the df-sane method for solving large-scale nonlinear
 systems of equations. For in depth information about all the parameters and the algorithm,
-see the paper [1].
+see [la2006spectral](@citet).
 
 ### Keyword Arguments
 
@@ -35,12 +35,6 @@ see the paper [1].
     ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and
     `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to
     ``||F||^2 / k^2``.
-
-### References
-
-[1] W LaCruz, JM Martinez, and M Raydan (2006), Spectral residual mathod without gradient
-information for solving large-scale nonlinear systems of equations, Mathematics of
-Computation, 75, 1429-1448.
 """
 @concrete struct SimpleDFSane{M} <: AbstractSimpleNonlinearSolveAlgorithm
     σ_min
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index 9351b45f1..680b9cd7c 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -1,7 +1,7 @@
 """
     SimpleKlement()
 
-A low-overhead implementation of [Klement](https://jatm.com.br/jatm/article/view/373). This
+A low-overhead implementation of `Klement` [klement2014using](@citep). This
 method is non-allocating on scalar and static array problems.
 """
 struct SimpleKlement <: AbstractSimpleNonlinearSolveAlgorithm end
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index eb7b1bbaf..9f8896ed1 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -9,18 +9,12 @@ If the threshold is larger than the problem size, then this method will use `Sim
 
 ### Keyword Arguments:
 
-  - `linesearch`: If `linesearch` is `Val(true)`, then we use the
-    `LiFukushimaLineSearch` [1] line search else no line search is used. For advanced
-    customization of the line search, use the [`LimitedMemoryBroyden`](@ref) algorithm in
+  - `linesearch`: If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch`
+    [li2000derivative](@cite) line search else no line search is used. For advanced
+    customization of the line search, use the `LimitedMemoryBroyden` algorithm in
     `NonlinearSolve.jl`.
   - `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we
     will compute the scaling using `2 * norm(fu) / max(norm(u), true)`.
-
-### References
-
-[1] Li, Dong-Hui, and Masao Fukushima. "A derivative-free line search and global convergence
-of Broyden-like method for nonlinear equations." Optimization methods and software 13.3
-(2000): 181-201.
 """
 @concrete struct SimpleLimitedMemoryBroyden{threshold, linesearch} <:
                  AbstractSimpleNonlinearSolveAlgorithm

From 6a022263490572abfe2deb412bbc4e5d770a98e1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Jan 2024 21:22:00 +0000
Subject: [PATCH 294/375] Bump actions/cache from 3 to 4

Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4.
- [Release notes](https://github.com/actions/cache/releases)
- [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md)
- [Commits](https://github.com/actions/cache/compare/v3...v4)

---
updated-dependencies:
- dependency-name: actions/cache
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 lib/SimpleNonlinearSolve/.github/workflows/CI.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index ba2974f47..8855ab61a 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -20,7 +20,7 @@ jobs:
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         env:
           cache-name: cache-artifacts
         with:

From f86134d316a55095193623764788ffb27047a9c4 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 5 Feb 2024 18:35:51 -0500
Subject: [PATCH 295/375] Patch Adjoint Sensitivity for Simple Nonlinear Solve
 Algorithms

---
 lib/SimpleNonlinearSolve/Project.toml         |  7 ++++--
 .../SimpleNonlinearSolveChainRulesCoreExt.jl  | 21 ++++++++++++++++++
 .../src/SimpleNonlinearSolve.jl               | 22 +++++++++++++------
 lib/SimpleNonlinearSolve/test/Project.toml    |  2 ++
 lib/SimpleNonlinearSolve/test/adjoint.jl      | 14 ++++++++++++
 lib/SimpleNonlinearSolve/test/runtests.jl     |  1 +
 6 files changed, 58 insertions(+), 9 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/adjoint.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index ca40b1ed6..7390adc2c 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.3.1"
+version = "1.3.2"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -19,16 +19,19 @@ SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [weakdeps]
+ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 
 [extensions]
+SimpleNonlinearSolveChainRulesCoreExt = "ChainRulesCore"
 SimpleNonlinearSolvePolyesterForwardDiffExt = "PolyesterForwardDiff"
 SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
 
 [compat]
 ADTypes = "0.2.6"
 ArrayInterface = "7"
+ChainRulesCore = "1"
 ConcreteStructs = "0.2"
 DiffEqBase = "6.126"
 FastClosures = "0.3"
@@ -39,6 +42,6 @@ MaybeInplace = "0.1"
 PrecompileTools = "1"
 Reexport = "1"
 SciMLBase = "2.7"
-StaticArraysCore = "1.4"
 StaticArrays = "1"
+StaticArraysCore = "1.4"
 julia = "1.9"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
new file mode 100644
index 000000000..23cd25f33
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
@@ -0,0 +1,21 @@
+module SimpleNonlinearSolveChainRulesCoreExt
+
+using ChainRulesCore, DiffEqBase, SciMLBase, SimpleNonlinearSolve
+
+# The expectation here is that no-one is using this directly inside a GPU kernel. We can
+# eventually lift this requirement using a custom adjoint
+function ChainRulesCore.rrule(::typeof(SimpleNonlinearSolve.__internal_solve_up),
+        prob::NonlinearProblem,
+        sensealg::Union{Nothing, DiffEqBase.AbstractSensitivityAlgorithm}, u0, u0_changed,
+        p, p_changed, alg, args...; kwargs...)
+    out, ∇internal = DiffEqBase._solve_adjoint(prob, sensealg, u0, p,
+        SciMLBase.ChainRulesOriginator(), alg, args...; kwargs...)
+    function ∇__internal_solve_up(Δ)
+        ∂f, ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
+        return (∂f, ∂prob, ∂sensealg, ∂u0, NoTangent(), ∂p, NoTangent(), ∂originator,
+            ∂args...)
+    end
+    return out, ∇__internal_solve_up
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 73eda9977..3f12d8888 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -50,18 +50,26 @@ include("ad.jl")
 ## Default algorithm
 
 # Set the default bracketing method to ITP
-function SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...)
-    return solve(prob, ITP(); kwargs...)
-end
-
-function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing,
-        args...; kwargs...)
+SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...) = solve(prob, ITP(); kwargs...)
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing, args...; kwargs...)
     return solve(prob, ITP(), args...; kwargs...)
 end
 
 # By Pass the highlevel checks for NonlinearProblem for Simple Algorithms
 function SciMLBase.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
-        args...; kwargs...)
+        args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...)
+    if sensealg === nothing && haskey(prob.kwargs, :sensealg)
+        sensealg = prob.kwargs[:sensealg]
+    end
+    new_u0 = u0 !== nothing ? u0 : prob.u0
+    new_p = p !== nothing ? p : prob.p
+    return __internal_solve_up(prob, sensealg, new_u0, u0 === nothing, new_p, p === nothing,
+        alg, args...; kwargs...)
+end
+
+function __internal_solve_up(_prob::NonlinearProblem, sensealg, u0, u0_changed, p,
+        p_changed, alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...)
+    prob = u0_changed || p_changed ? remake(_prob; u0, p) : _prob
     return SciMLBase.__solve(prob, alg, args...; kwargs...)
 end
 
diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
index cbf18f66c..f690826d0 100644
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -9,8 +9,10 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
+SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [compat]
 NonlinearProblemLibrary = "0.1.2"
diff --git a/lib/SimpleNonlinearSolve/test/adjoint.jl b/lib/SimpleNonlinearSolve/test/adjoint.jl
new file mode 100644
index 000000000..27c746981
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/adjoint.jl
@@ -0,0 +1,14 @@
+using ForwardDiff, SciMLSensitivity, SimpleNonlinearSolve, Test, Zygote
+
+@testset "Simple Adjoint Test" begin
+    ff(u, p) = u .^ 2 .- p
+
+    function solve_nlprob(p)
+        prob = NonlinearProblem{false}(ff, [1.0, 2.0], p)
+        return sum(abs2, solve(prob, SimpleNewtonRaphson()).u)
+    end
+
+    p = [3.0, 2.0]
+
+    @test only(Zygote.gradient(solve_nlprob, p)) ≈ ForwardDiff.gradient(solve_nlprob, p)
+end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 4de514642..c9ff996dc 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -15,6 +15,7 @@ end
         @time @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
         @time @safetestset "Least Squares Tests" include("least_squares.jl")
         @time @safetestset "23 Test Problems" include("23_test_problems.jl")
+        @time @safetestset "Simple Adjoint Tests" include("adjoint.jl")
     end
 
     if GROUP == "CUDA"

From 9bdd4858e2a9299ce7a0d4cc7a3f0141c87e874b Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 8 Feb 2024 10:49:08 -0500
Subject: [PATCH 296/375] Run tests in Parallel

---
 .../.github/workflows/CI.yml                  |  4 +-
 .../test/23_test_problems.jl                  | 66 ++++++++--------
 lib/SimpleNonlinearSolve/test/Project.toml    |  1 +
 lib/SimpleNonlinearSolve/test/adjoint.jl      |  4 +-
 lib/SimpleNonlinearSolve/test/basictests.jl   | 75 +++++++------------
 lib/SimpleNonlinearSolve/test/cuda.jl         | 26 +++----
 .../test/cuda/Project.toml                    |  1 +
 lib/SimpleNonlinearSolve/test/forward_ad.jl   |  4 +-
 .../test/least_squares.jl                     |  5 +-
 .../test/matrix_resizing_tests.jl             |  4 +-
 lib/SimpleNonlinearSolve/test/runtests.jl     | 18 ++---
 11 files changed, 96 insertions(+), 112 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index 8855ab61a..3bc260f64 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -32,9 +32,11 @@ jobs:
             ${{ runner.os }}-
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
+        with:
+          annotate: true
         env:
           GROUP: ${{ matrix.group }}
-          JULIA_NUM_THREADS: 11
+          JULIA_NUM_THREADS: "auto"
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v3
         with:
diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
index 18ac5e9bd..73bb304ec 100644
--- a/lib/SimpleNonlinearSolve/test/23_test_problems.jl
+++ b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
@@ -1,4 +1,4 @@
-using SimpleNonlinearSolve, LinearAlgebra, NonlinearProblemLibrary, DiffEqBase, Test
+using SimpleNonlinearSolve, LinearAlgebra, NonlinearProblemLibrary, DiffEqBase, XUnit
 
 problems = NonlinearProblemLibrary.problems
 dicts = NonlinearProblemLibrary.dicts
@@ -37,49 +37,51 @@ function test_on_library(problems, dicts, alg_ops, broken_tests, ϵ = 1e-4;
     end
 end
 
-@testset "SimpleNewtonRaphson 23 Test Problems" begin
-    alg_ops = (SimpleNewtonRaphson(),)
+@testset "23 Test Problems" begin
+    @testcase "SimpleNewtonRaphson 23 Test Problems" begin
+        alg_ops = (SimpleNewtonRaphson(),)
 
-    # dictionary with indices of test problems where method does not converge to small residual
-    broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = []
+        # dictionary with indices of test problems where method does not converge to small residual
+        broken_tests = Dict(alg => Int[] for alg in alg_ops)
+        broken_tests[alg_ops[1]] = []
 
-    test_on_library(problems, dicts, alg_ops, broken_tests)
-end
+        test_on_library(problems, dicts, alg_ops, broken_tests)
+    end
 
-@testset "SimpleTrustRegion 23 Test Problems" begin
-    alg_ops = (SimpleTrustRegion(),)
+    @testcase "SimpleTrustRegion 23 Test Problems" begin
+        alg_ops = (SimpleTrustRegion(),)
 
-    # dictionary with indices of test problems where method does not converge to small residual
-    broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [3, 6, 15, 16, 21]
+        # dictionary with indices of test problems where method does not converge to small residual
+        broken_tests = Dict(alg => Int[] for alg in alg_ops)
+        broken_tests[alg_ops[1]] = [3, 6, 15, 16, 21]
 
-    test_on_library(problems, dicts, alg_ops, broken_tests)
-end
+        test_on_library(problems, dicts, alg_ops, broken_tests)
+    end
 
-@testset "SimpleDFSane 23 Test Problems" begin
-    alg_ops = (SimpleDFSane(),)
+    @testcase "SimpleDFSane 23 Test Problems" begin
+        alg_ops = (SimpleDFSane(),)
 
-    broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 11, 21]
+        broken_tests = Dict(alg => Int[] for alg in alg_ops)
+        broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 11, 21]
 
-    test_on_library(problems, dicts, alg_ops, broken_tests)
-end
+        test_on_library(problems, dicts, alg_ops, broken_tests)
+    end
 
-@testset "SimpleBroyden 23 Test Problems" begin
-    alg_ops = (SimpleBroyden(),)
+    @testcase "SimpleBroyden 23 Test Problems" begin
+        alg_ops = (SimpleBroyden(),)
 
-    broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 5, 11]
+        broken_tests = Dict(alg => Int[] for alg in alg_ops)
+        broken_tests[alg_ops[1]] = [1, 5, 11]
 
-    test_on_library(problems, dicts, alg_ops, broken_tests)
-end
+        test_on_library(problems, dicts, alg_ops, broken_tests)
+    end
 
-@testset "SimpleKlement 23 Test Problems" begin
-    alg_ops = (SimpleKlement(),)
+    @testcase "SimpleKlement 23 Test Problems" begin
+        alg_ops = (SimpleKlement(),)
 
-    broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 11, 12, 22]
+        broken_tests = Dict(alg => Int[] for alg in alg_ops)
+        broken_tests[alg_ops[1]] = [1, 2, 4, 5, 11, 12, 22]
 
-    test_on_library(problems, dicts, alg_ops, broken_tests)
+        test_on_library(problems, dicts, alg_ops, broken_tests)
+    end
 end
diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
index f690826d0..9077234a6 100644
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/Project.toml
@@ -12,6 +12,7 @@ SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+XUnit = "3e3c03f2-1a94-11e9-2981-050a4ca824ab"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [compat]
diff --git a/lib/SimpleNonlinearSolve/test/adjoint.jl b/lib/SimpleNonlinearSolve/test/adjoint.jl
index 27c746981..cf2ef38d7 100644
--- a/lib/SimpleNonlinearSolve/test/adjoint.jl
+++ b/lib/SimpleNonlinearSolve/test/adjoint.jl
@@ -1,6 +1,6 @@
-using ForwardDiff, SciMLSensitivity, SimpleNonlinearSolve, Test, Zygote
+using ForwardDiff, SciMLSensitivity, SimpleNonlinearSolve, XUnit, Zygote
 
-@testset "Simple Adjoint Test" begin
+@testcase "Simple Adjoint Test" begin
     ff(u, p) = u .^ 2 .- p
 
     function solve_nlprob(p)
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index 5938197c7..c46230476 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -1,5 +1,5 @@
 using AllocCheck, LinearSolve, SimpleNonlinearSolve, StaticArrays, Random,
-    LinearAlgebra, Test, ForwardDiff, DiffEqBase
+    LinearAlgebra, XUnit, ForwardDiff, DiffEqBase
 import PolyesterForwardDiff
 
 _nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x))
@@ -25,33 +25,29 @@ const TERMINATION_CONDITIONS = [
     AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode(),
 ]
 
-# --- SimpleNewtonRaphson tests ---
-
-@testset "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion)
-    # Eval else the alg is type unstable
-    @eval begin
-        function benchmark_nlsolve_oop(f, u0, p = 2.0; autodiff = nothing)
-            prob = NonlinearProblem{false}(f, u0, p)
-            return solve(prob, $(alg)(; autodiff), abstol = 1e-9)
-        end
+function benchmark_nlsolve_oop(f::F, u0, p = 2.0; solver) where {F}
+    prob = NonlinearProblem{false}(f, u0, p)
+    return solve(prob, solver; abstol = 1e-9)
+end
+function benchmark_nlsolve_iip(f!::F, u0, p = 2.0; solver) where {F}
+    prob = NonlinearProblem{true}(f!, u0, p)
+    return solve(prob, solver; abstol = 1e-9)
+end
 
-        function benchmark_nlsolve_iip(f, u0, p = 2.0; autodiff = nothing)
-            prob = NonlinearProblem{true}(f, u0, p)
-            return solve(prob, $(alg)(; autodiff), abstol = 1e-9)
-        end
-    end
+# --- SimpleNewtonRaphson tests ---
 
+@testcase "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion)
     @testset "AutoDiff: $(_nameof(autodiff))" for autodiff in (AutoFiniteDiff(),
         AutoForwardDiff(), AutoPolyesterForwardDiff())
         @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
             u0 isa SVector && autodiff isa AutoPolyesterForwardDiff && continue
-            sol = benchmark_nlsolve_oop(quadratic_f, u0; autodiff)
+            sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = alg(; autodiff))
             @test SciMLBase.successful_retcode(sol)
             @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
         end
 
         @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
-            sol = benchmark_nlsolve_iip(quadratic_f!, u0; autodiff)
+            sol = benchmark_nlsolve_iip(quadratic_f!, u0; solver = alg(; autodiff))
             @test SciMLBase.successful_retcode(sol)
             @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
         end
@@ -67,16 +63,11 @@ end
 
 # --- SimpleHalley tests ---
 
-@testset "SimpleHalley" begin
-    function benchmark_nlsolve_oop(f, u0, p = 2.0; autodiff = nothing)
-        prob = NonlinearProblem{false}(f, u0, p)
-        return solve(prob, SimpleHalley(; autodiff), abstol = 1e-9)
-    end
-
+@testcase "SimpleHalley" begin
     @testset "AutoDiff: $(_nameof(autodiff))" for autodiff in (AutoFiniteDiff(),
         AutoForwardDiff())
         @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
-            sol = benchmark_nlsolve_oop(quadratic_f, u0; autodiff)
+            sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = SimpleHalley(; autodiff))
             @test SciMLBase.successful_retcode(sol)
             @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
         end
@@ -92,27 +83,17 @@ end
 
 # --- SimpleBroyden / SimpleKlement / SimpleLimitedMemoryBroyden tests ---
 
-@testset "$(_nameof(alg))" for alg in [SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
+@testcase "$(_nameof(alg))" for alg in [SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
     SimpleLimitedMemoryBroyden(), SimpleBroyden(; linesearch = Val(true)),
     SimpleLimitedMemoryBroyden(; linesearch = Val(true))]
-    function benchmark_nlsolve_oop(f, u0, p = 2.0)
-        prob = NonlinearProblem{false}(f, u0, p)
-        return solve(prob, alg, abstol = 1e-9)
-    end
-
-    function benchmark_nlsolve_iip(f, u0, p = 2.0)
-        prob = NonlinearProblem{true}(f, u0, p)
-        return solve(prob, alg, abstol = 1e-9)
-    end
-
     @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
-        sol = benchmark_nlsolve_oop(quadratic_f, u0)
+        sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = alg)
         @test SciMLBase.successful_retcode(sol)
         @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
     end
 
     @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
-        sol = benchmark_nlsolve_iip(quadratic_f!, u0)
+        sol = benchmark_nlsolve_iip(quadratic_f!, u0; solver = alg)
         @test SciMLBase.successful_retcode(sol)
         @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
     end
@@ -126,16 +107,11 @@ end
 end
 
 @testset "Newton Fails" begin
-    function benchmark_nlsolve_oop(f, u0, p, alg)
-        prob = NonlinearProblem{false}(f, u0, p)
-        return solve(prob, alg; abstol = 1e-9)
-    end
-
     u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
     p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
 
-    for alg in (SimpleDFSane(), SimpleTrustRegion(), SimpleHalley())
-        sol = benchmark_nlsolve_oop(newton_fails, u0, p, alg)
+    @testcase "$(alg)" for alg in (SimpleDFSane(), SimpleTrustRegion(), SimpleHalley())
+        sol = benchmark_nlsolve_oop(newton_fails, u0, p; solver = alg)
         @test SciMLBase.successful_retcode(sol)
         @test all(abs.(newton_fails(sol.u, p)) .< 1e-9)
     end
@@ -144,7 +120,7 @@ end
 # --- Allocation Checks ---
 
 ## SimpleDFSane needs to allocate a history vector
-@testset "Allocation Checks: $(_nameof(alg))" for alg in (SimpleNewtonRaphson(),
+@testcase "Allocation Checks: $(_nameof(alg))" for alg in (SimpleNewtonRaphson(),
     SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
     SimpleTrustRegion(), SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)),
     SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
@@ -173,7 +149,7 @@ end
 
 # --- Interval Nonlinear Problems ---
 
-@testset "Interval Nonlinear Problem: $(alg)" for alg in (Bisection(), Falsi(), Ridder(),
+@testcase "Interval Nonlinear Problem: $(alg)" for alg in (Bisection(), Falsi(), Ridder(),
     Brent(), ITP(), Alefeld())
     tspan = (1.0, 20.0)
 
@@ -215,7 +191,8 @@ end
     end
 end
 
-@testset "Tolerance Tests Interval Methods: $(alg)" for alg in (Bisection(), Falsi(), ITP())
+@testcase "Tolerance Tests Interval Methods: $(alg)" for alg in (Bisection(), Falsi(),
+    ITP())
     tspan = (1.0, 20.0)
     probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
     tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
@@ -228,7 +205,7 @@ end
     end
 end
 
-@testset "Tolerance Tests Interval Methods: $(alg)" for alg in (Ridder(), Brent())
+@testcase "Tolerance Tests Interval Methods: $(alg)" for alg in (Ridder(), Brent())
     tspan = (1.0, 20.0)
     probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
     tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution
@@ -241,7 +218,7 @@ end
     end
 end
 
-@testset "Flipped Signs and Reversed Tspan: $(alg)" for alg in (Alefeld(), Bisection(),
+@testcase "Flipped Signs and Reversed Tspan: $(alg)" for alg in (Alefeld(), Bisection(),
     Falsi(), Brent(), ITP(), Ridder())
     f1(u, p) = u * u - p
     f2(u, p) = p - u * u
diff --git a/lib/SimpleNonlinearSolve/test/cuda.jl b/lib/SimpleNonlinearSolve/test/cuda.jl
index fddf751de..7708ae5c4 100644
--- a/lib/SimpleNonlinearSolve/test/cuda.jl
+++ b/lib/SimpleNonlinearSolve/test/cuda.jl
@@ -1,4 +1,4 @@
-using SimpleNonlinearSolve, StaticArrays, CUDA, Test
+using SimpleNonlinearSolve, StaticArrays, CUDA, XUnit
 
 CUDA.allowscalar(false)
 
@@ -6,12 +6,10 @@ f(u, p) = u .* u .- 2
 f!(du, u, p) = du .= u .* u .- 2
 
 @testset "Solving on GPUs" begin
-    for alg in (SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(), SimpleBroyden(),
-        SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
-        SimpleBroyden(; linesearch = Val(true)),
+    @testcase "$(alg)" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
+        SimpleTrustRegion(), SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
+        SimpleHalley(), SimpleBroyden(; linesearch = Val(true)),
         SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
-        @info "Testing $alg on CUDA"
-
         # Static Arrays
         u0 = @SVector[1.0f0, 1.0f0]
         probN = NonlinearProblem{false}(f, u0)
@@ -27,12 +25,13 @@ f!(du, u, p) = du .= u .* u .- 2
         @test maximum(abs, sol.resid) ≤ 1.0f-6
 
         # Regular Arrays Inplace
-        alg isa SimpleHalley && continue
-        u0 = [1.0, 1.0]
-        probN = NonlinearProblem{true}(f!, u0)
-        sol = solve(probN, alg; abstol = 1.0f-6)
-        @test SciMLBase.successful_retcode(sol)
-        @test maximum(abs, sol.resid) ≤ 1.0f-6
+        if !(alg isa SimpleHalley)
+            u0 = [1.0, 1.0]
+            probN = NonlinearProblem{true}(f!, u0)
+            sol = solve(probN, alg; abstol = 1.0f-6)
+            @test SciMLBase.successful_retcode(sol)
+            @test maximum(abs, sol.resid) ≤ 1.0f-6
+        end
     end
 end
 
@@ -44,7 +43,8 @@ end
 @testset "CUDA Kernel Launch Test" begin
     prob = NonlinearProblem{false}(f, @SVector[1.0f0, 1.0f0])
 
-    for alg in (SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(), SimpleBroyden(),
+    @testcase "$(alg)" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
+        SimpleTrustRegion(), SimpleBroyden(),
         SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
         SimpleBroyden(; linesearch = Val(true)),
         SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
diff --git a/lib/SimpleNonlinearSolve/test/cuda/Project.toml b/lib/SimpleNonlinearSolve/test/cuda/Project.toml
index 36c63059b..4fdee40a7 100644
--- a/lib/SimpleNonlinearSolve/test/cuda/Project.toml
+++ b/lib/SimpleNonlinearSolve/test/cuda/Project.toml
@@ -3,3 +3,4 @@ CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+XUnit = "3e3c03f2-1a94-11e9-2981-050a4ca824ab"
diff --git a/lib/SimpleNonlinearSolve/test/forward_ad.jl b/lib/SimpleNonlinearSolve/test/forward_ad.jl
index f545ccb0c..78059c2f0 100644
--- a/lib/SimpleNonlinearSolve/test/forward_ad.jl
+++ b/lib/SimpleNonlinearSolve/test/forward_ad.jl
@@ -1,4 +1,4 @@
-using ForwardDiff, SimpleNonlinearSolve, StaticArrays, Test, LinearAlgebra
+using ForwardDiff, SimpleNonlinearSolve, StaticArrays, XUnit, LinearAlgebra
 import SimpleNonlinearSolve: AbstractSimpleNonlinearSolveAlgorithm
 
 test_f!(du, u, p) = (@. du = u^2 - p)
@@ -35,7 +35,7 @@ __compatible(::AbstractSimpleNonlinearSolveAlgorithm, ::Val{:iip}) = true
 __compatible(::AbstractSimpleNonlinearSolveAlgorithm, ::Val{:oop}) = true
 __compatible(::SimpleHalley, ::Val{:iip}) = false
 
-@testset "ForwardDiff.jl Integration: $(alg)" for alg in (SimpleNewtonRaphson(),
+@testcase "ForwardDiff.jl Integration: $(alg)" for alg in (SimpleNewtonRaphson(),
     SimpleTrustRegion(), SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleDFSane())
     us = (2.0, @SVector[1.0, 1.0], [1.0, 1.0], ones(2, 2), @SArray ones(2, 2))
 
diff --git a/lib/SimpleNonlinearSolve/test/least_squares.jl b/lib/SimpleNonlinearSolve/test/least_squares.jl
index bc801421f..6c48a9807 100644
--- a/lib/SimpleNonlinearSolve/test/least_squares.jl
+++ b/lib/SimpleNonlinearSolve/test/least_squares.jl
@@ -1,4 +1,4 @@
-using SimpleNonlinearSolve, LinearAlgebra, Test
+using SimpleNonlinearSolve, LinearAlgebra, XUnit
 
 true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
 
@@ -14,7 +14,8 @@ end
 θ_init = θ_true .+ 0.1
 prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
 
-for solver in [SimpleNewtonRaphson(AutoForwardDiff()), SimpleGaussNewton(AutoForwardDiff()),
+@testcase "Solver: $(solver)" for solver in [SimpleNewtonRaphson(AutoForwardDiff()),
+    SimpleGaussNewton(AutoForwardDiff()),
     SimpleNewtonRaphson(AutoFiniteDiff()), SimpleGaussNewton(AutoFiniteDiff())]
     sol = solve(prob_oop, solver)
     @test norm(sol.resid) < 1e-12
diff --git a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
index 455aac91a..63a0a1735 100644
--- a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
@@ -1,4 +1,4 @@
-using SimpleNonlinearSolve
+using SimpleNonlinearSolve, XUnit
 
 ff(u, p) = u .* u .- p
 u0 = ones(2, 3)
@@ -6,7 +6,7 @@ p = 2.0
 vecprob = NonlinearProblem(ff, vec(u0), p)
 prob = NonlinearProblem(ff, u0, p)
 
-@testset "$(alg)" for alg in (SimpleKlement(), SimpleBroyden(), SimpleNewtonRaphson(),
+@testcase "$(alg)" for alg in (SimpleKlement(), SimpleBroyden(), SimpleNewtonRaphson(),
     SimpleDFSane(), SimpleLimitedMemoryBroyden(; threshold = Val(2)), SimpleTrustRegion())
     @test vec(solve(prob, alg).u) ≈ solve(vecprob, alg).u
 end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index c9ff996dc..5d9216ae3 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -1,4 +1,4 @@
-using Pkg, SafeTestsets, Test
+using Pkg, SafeTestsets, XUnit
 
 const GROUP = get(ENV, "GROUP", "All")
 
@@ -8,18 +8,18 @@ function activate_env(env)
     Pkg.instantiate()
 end
 
-@time @testset "SimpleNonlinearSolve.jl" begin
+@testset runner=ParallelTestRunner() "SimpleNonlinearSolve.jl" begin
     if GROUP == "All" || GROUP == "Core"
-        @time @safetestset "Basic Tests" include("basictests.jl")
-        @time @safetestset "Forward AD" include("forward_ad.jl")
-        @time @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
-        @time @safetestset "Least Squares Tests" include("least_squares.jl")
-        @time @safetestset "23 Test Problems" include("23_test_problems.jl")
-        @time @safetestset "Simple Adjoint Tests" include("adjoint.jl")
+        @safetestset "Basic Tests" include("basictests.jl")
+        @safetestset "Forward AD" include("forward_ad.jl")
+        @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
+        @safetestset "Least Squares Tests" include("least_squares.jl")
+        @safetestset "23 Test Problems" include("23_test_problems.jl")
+        @safetestset "Simple Adjoint Tests" include("adjoint.jl")
     end
 
     if GROUP == "CUDA"
         activate_env("cuda")
-        @time @safetestset "CUDA Tests" include("cuda.jl")
+        @safetestset "CUDA Tests" include("cuda.jl")
     end
 end

From 6ac95bda6a2838ad054531c548554df417243451 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 8 Feb 2024 13:43:29 -0500
Subject: [PATCH 297/375] Add NLsolve update rule to SimpleTrustRegion

---
 lib/SimpleNonlinearSolve/Project.toml         |  2 +-
 .../src/nlsolve/trustRegion.jl                | 68 ++++++++++++++-----
 .../test/23_test_problems.jl                  |  6 +-
 lib/SimpleNonlinearSolve/test/basictests.jl   | 10 ++-
 lib/SimpleNonlinearSolve/test/cuda.jl         |  7 +-
 lib/SimpleNonlinearSolve/test/forward_ad.jl   |  3 +-
 .../test/matrix_resizing_tests.jl             |  3 +-
 7 files changed, 72 insertions(+), 27 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 7390adc2c..b0cbd3739 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.3.2"
+version = "1.4.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index e8a90cdbd..03c4692e7 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -1,8 +1,9 @@
 """
-    SimpleTrustRegion(; autodiff = AutoForwardDiff(), max_trust_radius::Real = 0.0,
-        initial_trust_radius::Real = 0.0, step_threshold::Real = 0.1,
-        shrink_threshold::Real = 0.25, expand_threshold::Real = 0.75,
-        shrink_factor::Real = 0.25, expand_factor::Real = 2.0, max_shrink_times::Int = 32)
+    SimpleTrustRegion(; autodiff = AutoForwardDiff(), max_trust_radius = 0.0,
+        initial_trust_radius = 0.0, step_threshold = nothing,
+        shrink_threshold = nothing, expand_threshold = nothing,
+        shrink_factor = 0.25, expand_factor = 2.0, max_shrink_times::Int = 32,
+        nlsolve_update_rule = Val(false))
 
 A low-overhead implementation of a trust-region solver. This method is non-allocating on
 scalar and static array problems.
@@ -36,17 +37,22 @@ scalar and static array problems.
     `expand_threshold < r` (with `r` defined in `shrink_threshold`). Defaults to `2.0`.
   - `max_shrink_times`: the maximum number of times to shrink the trust region radius in a
     row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`.
+  - `nlsolve_update_rule`: If set to `Val(true)`, updates the trust region radius using the
+    update rule from NLSolve.jl. Defaults to `Val(false)`. If set to `Val(true)`, few of the
+    radius update parameters -- `step_threshold = 0.05`, `expand_threshold = 0.9`, and
+    `shrink_factor = 0.5` -- have different defaults.
 """
 @kwdef @concrete struct SimpleTrustRegion <: AbstractNewtonAlgorithm
     autodiff = nothing
     max_trust_radius = 0.0
     initial_trust_radius = 0.0
     step_threshold = 0.0001
-    shrink_threshold = 0.25
-    expand_threshold = 0.75
-    shrink_factor = 0.25
+    shrink_threshold = nothing
+    expand_threshold = nothing
+    shrink_factor = nothing
     expand_factor = 2.0
     max_shrink_times::Int = 32
+    nlsolve_update_rule = Val(false)
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args...;
@@ -57,14 +63,27 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
     Δₘₐₓ = T(alg.max_trust_radius)
     Δ = T(alg.initial_trust_radius)
     η₁ = T(alg.step_threshold)
-    η₂ = T(alg.shrink_threshold)
-    η₃ = T(alg.expand_threshold)
-    t₁ = T(alg.shrink_factor)
+    if alg.shrink_threshold === nothing
+        η₂ = _unwrap_val(alg.nlsolve_update_rule) ? T(0.05) : T(0.25)
+    else
+        η₂ = T(alg.shrink_threshold)
+    end
+    if alg.expand_threshold === nothing
+        η₃ = _unwrap_val(alg.nlsolve_update_rule) ? T(0.9) : T(0.75)
+    else
+        η₃ = T(alg.expand_threshold)
+    end
+    if alg.shrink_factor === nothing
+        t₁ = _unwrap_val(alg.nlsolve_update_rule) ? T(0.5) : T(0.25)
+    else
+        t₁ = T(alg.shrink_factor)
+    end
     t₂ = T(alg.expand_factor)
     max_shrink_times = alg.max_shrink_times
     autodiff = __get_concrete_autodiff(prob, alg.autodiff)
 
     fx = _get_fx(prob, x)
+    norm_fx = norm(fx)
     @bb xo = copy(x)
     J, jac_cache = jacobian_cache(autodiff, prob.f, fx, x, prob.p)
     fx, ∇f = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
@@ -73,10 +92,17 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
         termination_condition)
 
     # Set default trust region radius if not specified by user.
-    Δₘₐₓ == 0 && (Δₘₐₓ = max(norm(fx), maximum(x) - minimum(x)))
-    Δ == 0 && (Δ = Δₘₐₓ / 11)
+    Δₘₐₓ == 0 && (Δₘₐₓ = max(norm_fx, maximum(x) - minimum(x)))
+    if Δ == 0
+        if _unwrap_val(alg.nlsolve_update_rule)
+            norm_x = norm(x)
+            Δ = T(ifelse(norm_x > 0, norm_x, 1))
+        else
+            Δ = T(Δₘₐₓ / 11)
+        end
+    end
 
-    fₖ = 0.5 * norm(fx)^2
+    fₖ = 0.5 * norm_fx^2
     H = ∇f' * ∇f
     g = _restructure(x, ∇f' * _vec(fx))
     shrink_counter = 0
@@ -87,7 +113,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
     @bb Hδ = copy(x)
     dogleg_cache = (; δsd, δN_δsd, δN)
 
-    for k in 1:maxiters
+    for _ in 1:maxiters
         # Solve the trust region subproblem.
         δ = dogleg_method!!(dogleg_cache, ∇f, fx, g, Δ)
         @bb @. x = xo + δ
@@ -107,7 +133,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
             Δ = t₁ * Δ
             shrink_counter += 1
             shrink_counter > max_shrink_times && return build_solution(prob, alg, x, fx;
-                retcode = ReturnCode.ConvergenceFailure)
+                retcode = ReturnCode.ShrinkThresholdExceeded)
         end
 
         if r ≥ η₁
@@ -121,12 +147,22 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
             fx, ∇f = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
             # Update the trust region radius.
-            (r > η₃) && (norm(δ) ≈ Δ) && (Δ = min(t₂ * Δ, Δₘₐₓ))
+            if !_unwrap_val(alg.nlsolve_update_rule) && r > η₃
+                Δ = min(t₂ * Δ, Δₘₐₓ)
+            end
             fₖ = fₖ₊₁
 
             @bb H = transpose(∇f) × ∇f
             @bb g = transpose(∇f) × vec(fx)
         end
+
+        if _unwrap_val(alg.nlsolve_update_rule)
+            if r > η₃
+                Δ = t₂ * norm(δ)
+            elseif r > 0.5
+                Δ = max(Δ, t₂ * norm(δ))
+            end
+        end
     end
 
     return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
index 73bb304ec..c04998281 100644
--- a/lib/SimpleNonlinearSolve/test/23_test_problems.jl
+++ b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
@@ -49,11 +49,13 @@ end
     end
 
     @testcase "SimpleTrustRegion 23 Test Problems" begin
-        alg_ops = (SimpleTrustRegion(),)
+        alg_ops = (SimpleTrustRegion(),
+            SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
 
         # dictionary with indices of test problems where method does not converge to small residual
         broken_tests = Dict(alg => Int[] for alg in alg_ops)
-        broken_tests[alg_ops[1]] = [3, 6, 15, 16, 21]
+        broken_tests[alg_ops[1]] = [3, 15, 16, 21]
+        broken_tests[alg_ops[2]] = [15, 16]
 
         test_on_library(problems, dicts, alg_ops, broken_tests)
     end
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
index c46230476..3f59dfb45 100644
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ b/lib/SimpleNonlinearSolve/test/basictests.jl
@@ -36,7 +36,9 @@ end
 
 # --- SimpleNewtonRaphson tests ---
 
-@testcase "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion)
+@testcase "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion,
+    (args...; kwargs...) -> SimpleTrustRegion(args...; nlsolve_update_rule = Val(true),
+        kwargs...))
     @testset "AutoDiff: $(_nameof(autodiff))" for autodiff in (AutoFiniteDiff(),
         AutoForwardDiff(), AutoPolyesterForwardDiff())
         @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
@@ -110,7 +112,8 @@ end
     u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
     p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
 
-    @testcase "$(alg)" for alg in (SimpleDFSane(), SimpleTrustRegion(), SimpleHalley())
+    @testcase "$(alg)" for alg in (SimpleDFSane(), SimpleTrustRegion(), SimpleHalley(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
         sol = benchmark_nlsolve_oop(newton_fails, u0, p; solver = alg)
         @test SciMLBase.successful_retcode(sol)
         @test all(abs.(newton_fails(sol.u, p)) .< 1e-9)
@@ -122,7 +125,8 @@ end
 ## SimpleDFSane needs to allocate a history vector
 @testcase "Allocation Checks: $(_nameof(alg))" for alg in (SimpleNewtonRaphson(),
     SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
-    SimpleTrustRegion(), SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)),
+    SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+    SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)),
     SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
     @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
 
diff --git a/lib/SimpleNonlinearSolve/test/cuda.jl b/lib/SimpleNonlinearSolve/test/cuda.jl
index 7708ae5c4..f1f372f6a 100644
--- a/lib/SimpleNonlinearSolve/test/cuda.jl
+++ b/lib/SimpleNonlinearSolve/test/cuda.jl
@@ -7,7 +7,8 @@ f!(du, u, p) = du .= u .* u .- 2
 
 @testset "Solving on GPUs" begin
     @testcase "$(alg)" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
-        SimpleTrustRegion(), SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
+        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+        SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
         SimpleHalley(), SimpleBroyden(; linesearch = Val(true)),
         SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
         # Static Arrays
@@ -44,8 +45,8 @@ end
     prob = NonlinearProblem{false}(f, @SVector[1.0f0, 1.0f0])
 
     @testcase "$(alg)" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
-        SimpleTrustRegion(), SimpleBroyden(),
-        SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
+        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+        SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
         SimpleBroyden(; linesearch = Val(true)),
         SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
         @test begin
diff --git a/lib/SimpleNonlinearSolve/test/forward_ad.jl b/lib/SimpleNonlinearSolve/test/forward_ad.jl
index 78059c2f0..99b32c3cb 100644
--- a/lib/SimpleNonlinearSolve/test/forward_ad.jl
+++ b/lib/SimpleNonlinearSolve/test/forward_ad.jl
@@ -36,7 +36,8 @@ __compatible(::AbstractSimpleNonlinearSolveAlgorithm, ::Val{:oop}) = true
 __compatible(::SimpleHalley, ::Val{:iip}) = false
 
 @testcase "ForwardDiff.jl Integration: $(alg)" for alg in (SimpleNewtonRaphson(),
-    SimpleTrustRegion(), SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleDFSane())
+    SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+    SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleDFSane())
     us = (2.0, @SVector[1.0, 1.0], [1.0, 1.0], ones(2, 2), @SArray ones(2, 2))
 
     @testset "Scalar AD" begin
diff --git a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
index 63a0a1735..778ec5185 100644
--- a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
@@ -7,6 +7,7 @@ vecprob = NonlinearProblem(ff, vec(u0), p)
 prob = NonlinearProblem(ff, u0, p)
 
 @testcase "$(alg)" for alg in (SimpleKlement(), SimpleBroyden(), SimpleNewtonRaphson(),
-    SimpleDFSane(), SimpleLimitedMemoryBroyden(; threshold = Val(2)), SimpleTrustRegion())
+    SimpleDFSane(), SimpleLimitedMemoryBroyden(; threshold = Val(2)), SimpleTrustRegion(),
+    SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
     @test vec(solve(prob, alg).u) ≈ solve(vecprob, alg).u
 end

From 1d8161837a6313067bf9e1baf79128f9a4b245a5 Mon Sep 17 00:00:00 2001
From: Avik Pal <avik.pal.2017@gmail.com>
Date: Thu, 8 Feb 2024 14:16:47 -0500
Subject: [PATCH 298/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 6dc0389c5..2d6167a0d 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -37,7 +37,7 @@ DiffEqBase = "6.144"
 FastClosures = "0.3"
 FiniteDiff = "2.21"
 ForwardDiff = "0.10.36"
-LinearAlgebra = "1.9"
+LinearAlgebra = "1.10"
 MaybeInplace = "0.1.1"
 PrecompileTools = "1.2"
 Reexport = "1.2"

From 03bab8ea1ef4e738fc379825ba8914fcde81868e Mon Sep 17 00:00:00 2001
From: Avik Pal <avik.pal.2017@gmail.com>
Date: Thu, 8 Feb 2024 14:19:02 -0500
Subject: [PATCH 299/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 2d6167a0d..f6d3ba7d5 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -42,6 +42,6 @@ MaybeInplace = "0.1.1"
 PrecompileTools = "1.2"
 Reexport = "1.2"
 SciMLBase = "2.23"
-StaticArrays = "1"
+StaticArrays = "1.8"
 StaticArraysCore = "1.4.2"
 julia = "1.10"

From 28a3c9b1d25187f66fb357019d8a87dbcaa7183d Mon Sep 17 00:00:00 2001
From: Avik Pal <avik.pal.2017@gmail.com>
Date: Thu, 8 Feb 2024 14:20:45 -0500
Subject: [PATCH 300/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index f6d3ba7d5..b2fb3baac 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -31,7 +31,7 @@ SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
 [compat]
 ADTypes = "0.2.6"
 ArrayInterface = "7.7"
-ChainRulesCore = "1"
+ChainRulesCore = "1.18"
 ConcreteStructs = "0.2.2"
 DiffEqBase = "6.144"
 FastClosures = "0.3"

From fe60a39808615c41fad422398971eff88c9627d5 Mon Sep 17 00:00:00 2001
From: Avik Pal <avik.pal.2017@gmail.com>
Date: Thu, 8 Feb 2024 14:44:31 -0500
Subject: [PATCH 301/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index b2fb3baac..fc4f75e71 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -33,7 +33,7 @@ ADTypes = "0.2.6"
 ArrayInterface = "7.7"
 ChainRulesCore = "1.18"
 ConcreteStructs = "0.2.2"
-DiffEqBase = "6.144"
+DiffEqBase = "6.146"
 FastClosures = "0.3"
 FiniteDiff = "2.21"
 ForwardDiff = "0.10.36"

From cd4899dab84e736f3d5d92a763e4395cd524a2dd Mon Sep 17 00:00:00 2001
From: Avik Pal <avik.pal.2017@gmail.com>
Date: Thu, 8 Feb 2024 16:13:56 -0500
Subject: [PATCH 302/375] Apply suggestions from code review

---
 lib/SimpleNonlinearSolve/Project.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index fc4f75e71..f5445b9d7 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -31,17 +31,17 @@ SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
 [compat]
 ADTypes = "0.2.6"
 ArrayInterface = "7.7"
-ChainRulesCore = "1.18"
+ChainRulesCore = "1.21"
 ConcreteStructs = "0.2.2"
 DiffEqBase = "6.146"
 FastClosures = "0.3"
-FiniteDiff = "2.21"
+FiniteDiff = "2.22"
 ForwardDiff = "0.10.36"
 LinearAlgebra = "1.10"
 MaybeInplace = "0.1.1"
 PrecompileTools = "1.2"
 Reexport = "1.2"
 SciMLBase = "2.23"
-StaticArrays = "1.8"
+StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"
 julia = "1.10"

From 07ec5739af11de447a5651c6636304986e12661d Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 9 Feb 2024 09:37:28 -0500
Subject: [PATCH 303/375] Use ReTestItems instead of XUnit

---
 .../.buildkite/pipeline.yml                   |  11 +-
 .../.github/workflows/CI.yml                  |   4 +-
 .../.github/workflows/Downstream.yml          |   4 +
 .../.github/workflows/FormatPR.yml            |   2 +-
 lib/SimpleNonlinearSolve/Project.toml         |  25 +-
 .../test/23_test_problems.jl                  |  89 ------
 lib/SimpleNonlinearSolve/test/Project.toml    |  19 --
 lib/SimpleNonlinearSolve/test/basictests.jl   | 268 -----------------
 .../test/core/23_test_problems_tests.jl       |  89 ++++++
 .../{adjoint.jl => core/adjoint_tests.jl}     |   4 +-
 .../test/core/forward_ad_tests.jl             |  90 ++++++
 .../test/core/least_squares_tests.jl          |  24 ++
 .../test/core/matrix_resizing_tests.jl        |  14 +
 .../test/core/rootfind_tests.jl               | 279 ++++++++++++++++++
 .../test/cuda/Project.toml                    |   6 -
 lib/SimpleNonlinearSolve/test/forward_ad.jl   |  83 ------
 .../test/{cuda.jl => gpu/cuda_tests.jl}       |  31 +-
 .../test/least_squares.jl                     |  22 --
 .../test/matrix_resizing_tests.jl             |  13 -
 lib/SimpleNonlinearSolve/test/runtests.jl     |  26 +-
 20 files changed, 564 insertions(+), 539 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/test/23_test_problems.jl
 delete mode 100644 lib/SimpleNonlinearSolve/test/Project.toml
 delete mode 100644 lib/SimpleNonlinearSolve/test/basictests.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
 rename lib/SimpleNonlinearSolve/test/{adjoint.jl => core/adjoint_tests.jl} (73%)
 create mode 100644 lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
 delete mode 100644 lib/SimpleNonlinearSolve/test/cuda/Project.toml
 delete mode 100644 lib/SimpleNonlinearSolve/test/forward_ad.jl
 rename lib/SimpleNonlinearSolve/test/{cuda.jl => gpu/cuda_tests.jl} (75%)
 delete mode 100644 lib/SimpleNonlinearSolve/test/least_squares.jl
 delete mode 100644 lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl

diff --git a/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
index e84559d00..ef04f99a6 100644
--- a/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
+++ b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
@@ -4,6 +4,12 @@ steps:
       - JuliaCI/julia#v1:
           version: "1"
       - JuliaCI/julia-test#v1:
+          coverage: true
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+          dirs:
+            - src
+            - ext
     agents:
       queue: "juliagpu"
       cuda: "*"
@@ -13,4 +19,7 @@ steps:
 
 env:
   GROUP: CUDA
-  JULIA_PKG_SERVER: ""
\ No newline at end of file
+  JULIA_PKG_SERVER: ""
+  RETESTITEMS_NWORKERS: 4
+  RETESTITEMS_NWORKER_THREADS: 2
+  SECRET_CODECOV_TOKEN: "HsC3X/h3CrsRhMR199BUbw9xmASfTTEtdT2JMJlbkW6nsMtZXDIWxKzBNOpn0123zn3K+VE30i7+aLpcqZjlux0BQDMVLx5O5KudjrJmBE2G8KMMv1pra+UHEOFdFP3TzjHXGcHzpUrjpG8if8cHbsWdyQCV0Hdx2qAPnhX5haGPyMuIlRfoaWQfdxJPl5fDLXs1xe0LnMcYMx8uUsvvJZ/hhFtMYWJU0WFtqnAkCR8h/pQd6yZOaGP2SXJkOR8+1xbx+M8m6agH9idp2BjDPpXMOo27V3O2Gkoy3R4b5ouLdqOMaZSIOemoYCv6oh+EkbKaFvZydvm0xCW5YBFPPw==;U2FsdGVkX19FT0yFV6EMY/NVSQslXE6byckN0qa/HVU5dd3d4swmOCWBkBPtemRPGvCMP669iXSPfDTlw7ZSvw=="
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index 3bc260f64..009bccfbf 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -36,7 +36,9 @@ jobs:
           annotate: true
         env:
           GROUP: ${{ matrix.group }}
-          JULIA_NUM_THREADS: "auto"
+          JULIA_NUM_THREADS: 11
+          RETESTITEMS_NWORKERS: 4
+          RETESTITEMS_NWORKER_THREADS: 2
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v3
         with:
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
index affe2fd97..a3c5003c1 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
@@ -24,6 +24,7 @@ jobs:
           - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceIII}
           - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceIV}
           - {user: SciML, repo: OrdinaryDiffEq.jl, group: InterfaceV}
+          - {user: SciML, repo: NonlinearSolve.jl, group: All}
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
@@ -53,6 +54,9 @@ jobs:
             @info "Not compatible with this release. No problem." exception=err
             exit(0)  # Exit immediately, as a success
           end
+        env:
+          RETESTITEMS_NWORKERS: 4
+          RETESTITEMS_NWORKER_THREADS: 2
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v3
         with:
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml
index eea22ebce..05caa2ebc 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatPR.yml
@@ -15,7 +15,7 @@ jobs:
       # https://github.com/peter-evans/create-pull-request#reference-example
       - name: Create Pull Request
         id: cpr
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: Format .jl files
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index b0cbd3739..6d6d5a55f 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.4.0"
+version = "1.4.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -41,7 +41,28 @@ LinearAlgebra = "1.9"
 MaybeInplace = "0.1"
 PrecompileTools = "1"
 Reexport = "1"
-SciMLBase = "2.7"
+SciMLBase = "2.23"
 StaticArrays = "1"
 StaticArraysCore = "1.4"
 julia = "1.9"
+
+[extras]
+AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
+NonlinearProblemLibrary = "b7050fa9-e91f-4b37-bcee-a89a063da141"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
+SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
+
+[targets]
+test = ["AllocCheck", "DiffEqBase", "ForwardDiff", "LinearAlgebra", "LinearSolve", "NonlinearProblemLibrary", "Pkg", "Random", "ReTestItems", "SciMLSensitivity", "StaticArrays", "Zygote", "CUDA", "PolyesterForwardDiff", "Reexport", "Test"]
diff --git a/lib/SimpleNonlinearSolve/test/23_test_problems.jl b/lib/SimpleNonlinearSolve/test/23_test_problems.jl
deleted file mode 100644
index c04998281..000000000
--- a/lib/SimpleNonlinearSolve/test/23_test_problems.jl
+++ /dev/null
@@ -1,89 +0,0 @@
-using SimpleNonlinearSolve, LinearAlgebra, NonlinearProblemLibrary, DiffEqBase, XUnit
-
-problems = NonlinearProblemLibrary.problems
-dicts = NonlinearProblemLibrary.dicts
-
-function test_on_library(problems, dicts, alg_ops, broken_tests, ϵ = 1e-4;
-        skip_tests = nothing)
-    for (idx, (problem, dict)) in enumerate(zip(problems, dicts))
-        x = dict["start"]
-        res = similar(x)
-        nlprob = NonlinearProblem(problem, copy(x))
-        @testset "$idx: $(dict["title"])" begin
-            for alg in alg_ops
-                try
-                    sol = solve(nlprob, alg;
-                        termination_condition = AbsNormTerminationMode())
-                    problem(res, sol.u, nothing)
-
-                    skip = skip_tests !== nothing && idx in skip_tests[alg]
-                    if skip
-                        @test_skip norm(res) ≤ ϵ
-                        continue
-                    end
-                    broken = idx in broken_tests[alg] ? true : false
-                    @test norm(res)≤ϵ broken=broken
-                catch e
-                    @error e
-                    broken = idx in broken_tests[alg] ? true : false
-                    if broken
-                        @test false broken=true
-                    else
-                        @test 1 == 2
-                    end
-                end
-            end
-        end
-    end
-end
-
-@testset "23 Test Problems" begin
-    @testcase "SimpleNewtonRaphson 23 Test Problems" begin
-        alg_ops = (SimpleNewtonRaphson(),)
-
-        # dictionary with indices of test problems where method does not converge to small residual
-        broken_tests = Dict(alg => Int[] for alg in alg_ops)
-        broken_tests[alg_ops[1]] = []
-
-        test_on_library(problems, dicts, alg_ops, broken_tests)
-    end
-
-    @testcase "SimpleTrustRegion 23 Test Problems" begin
-        alg_ops = (SimpleTrustRegion(),
-            SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
-
-        # dictionary with indices of test problems where method does not converge to small residual
-        broken_tests = Dict(alg => Int[] for alg in alg_ops)
-        broken_tests[alg_ops[1]] = [3, 15, 16, 21]
-        broken_tests[alg_ops[2]] = [15, 16]
-
-        test_on_library(problems, dicts, alg_ops, broken_tests)
-    end
-
-    @testcase "SimpleDFSane 23 Test Problems" begin
-        alg_ops = (SimpleDFSane(),)
-
-        broken_tests = Dict(alg => Int[] for alg in alg_ops)
-        broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 11, 21]
-
-        test_on_library(problems, dicts, alg_ops, broken_tests)
-    end
-
-    @testcase "SimpleBroyden 23 Test Problems" begin
-        alg_ops = (SimpleBroyden(),)
-
-        broken_tests = Dict(alg => Int[] for alg in alg_ops)
-        broken_tests[alg_ops[1]] = [1, 5, 11]
-
-        test_on_library(problems, dicts, alg_ops, broken_tests)
-    end
-
-    @testcase "SimpleKlement 23 Test Problems" begin
-        alg_ops = (SimpleKlement(),)
-
-        broken_tests = Dict(alg => Int[] for alg in alg_ops)
-        broken_tests[alg_ops[1]] = [1, 2, 4, 5, 11, 12, 22]
-
-        test_on_library(problems, dicts, alg_ops, broken_tests)
-    end
-end
diff --git a/lib/SimpleNonlinearSolve/test/Project.toml b/lib/SimpleNonlinearSolve/test/Project.toml
deleted file mode 100644
index 9077234a6..000000000
--- a/lib/SimpleNonlinearSolve/test/Project.toml
+++ /dev/null
@@ -1,19 +0,0 @@
-[deps]
-AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
-DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
-ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
-LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
-NonlinearProblemLibrary = "b7050fa9-e91f-4b37-bcee-a89a063da141"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
-Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
-SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-XUnit = "3e3c03f2-1a94-11e9-2981-050a4ca824ab"
-Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
-
-[compat]
-NonlinearProblemLibrary = "0.1.2"
diff --git a/lib/SimpleNonlinearSolve/test/basictests.jl b/lib/SimpleNonlinearSolve/test/basictests.jl
deleted file mode 100644
index 3f59dfb45..000000000
--- a/lib/SimpleNonlinearSolve/test/basictests.jl
+++ /dev/null
@@ -1,268 +0,0 @@
-using AllocCheck, LinearSolve, SimpleNonlinearSolve, StaticArrays, Random,
-    LinearAlgebra, XUnit, ForwardDiff, DiffEqBase
-import PolyesterForwardDiff
-
-_nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x))
-
-quadratic_f(u, p) = u .* u .- p
-quadratic_f!(du, u, p) = (du .= u .* u .- p)
-quadratic_f2(u, p) = @. p[1] * u * u - p[2]
-
-function newton_fails(u, p)
-    return 0.010000000000000002 .+
-           10.000000000000002 ./ (1 .+
-            (0.21640425613334457 .+
-             216.40425613334457 ./ (1 .+
-              (0.21640425613334457 .+
-               216.40425613334457 ./
-               (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
-           0.0011552453009332421u .- p
-end
-
-const TERMINATION_CONDITIONS = [
-    NormTerminationMode(), RelTerminationMode(), RelNormTerminationMode(),
-    AbsTerminationMode(), AbsNormTerminationMode(), RelSafeTerminationMode(),
-    AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode(),
-]
-
-function benchmark_nlsolve_oop(f::F, u0, p = 2.0; solver) where {F}
-    prob = NonlinearProblem{false}(f, u0, p)
-    return solve(prob, solver; abstol = 1e-9)
-end
-function benchmark_nlsolve_iip(f!::F, u0, p = 2.0; solver) where {F}
-    prob = NonlinearProblem{true}(f!, u0, p)
-    return solve(prob, solver; abstol = 1e-9)
-end
-
-# --- SimpleNewtonRaphson tests ---
-
-@testcase "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion,
-    (args...; kwargs...) -> SimpleTrustRegion(args...; nlsolve_update_rule = Val(true),
-        kwargs...))
-    @testset "AutoDiff: $(_nameof(autodiff))" for autodiff in (AutoFiniteDiff(),
-        AutoForwardDiff(), AutoPolyesterForwardDiff())
-        @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
-            u0 isa SVector && autodiff isa AutoPolyesterForwardDiff && continue
-            sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = alg(; autodiff))
-            @test SciMLBase.successful_retcode(sol)
-            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
-        end
-
-        @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
-            sol = benchmark_nlsolve_iip(quadratic_f!, u0; solver = alg(; autodiff))
-            @test SciMLBase.successful_retcode(sol)
-            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
-        end
-    end
-
-    @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
-        u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
-
-        probN = NonlinearProblem(quadratic_f, u0, 2.0)
-        @test all(solve(probN, alg(); termination_condition).u .≈ sqrt(2.0))
-    end
-end
-
-# --- SimpleHalley tests ---
-
-@testcase "SimpleHalley" begin
-    @testset "AutoDiff: $(_nameof(autodiff))" for autodiff in (AutoFiniteDiff(),
-        AutoForwardDiff())
-        @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
-            sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = SimpleHalley(; autodiff))
-            @test SciMLBase.successful_retcode(sol)
-            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
-        end
-    end
-
-    @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
-        u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
-
-        probN = NonlinearProblem(quadratic_f, u0, 2.0)
-        @test all(solve(probN, SimpleHalley(); termination_condition).u .≈ sqrt(2.0))
-    end
-end
-
-# --- SimpleBroyden / SimpleKlement / SimpleLimitedMemoryBroyden tests ---
-
-@testcase "$(_nameof(alg))" for alg in [SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
-    SimpleLimitedMemoryBroyden(), SimpleBroyden(; linesearch = Val(true)),
-    SimpleLimitedMemoryBroyden(; linesearch = Val(true))]
-    @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
-        sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = alg)
-        @test SciMLBase.successful_retcode(sol)
-        @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
-    end
-
-    @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
-        sol = benchmark_nlsolve_iip(quadratic_f!, u0; solver = alg)
-        @test SciMLBase.successful_retcode(sol)
-        @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
-    end
-
-    @testset "Termination condition: $(termination_condition) u0: $(_nameof(u0))" for termination_condition in TERMINATION_CONDITIONS,
-        u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
-
-        probN = NonlinearProblem(quadratic_f, u0, 2.0)
-        @test all(solve(probN, alg; termination_condition).u .≈ sqrt(2.0))
-    end
-end
-
-@testset "Newton Fails" begin
-    u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
-    p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
-
-    @testcase "$(alg)" for alg in (SimpleDFSane(), SimpleTrustRegion(), SimpleHalley(),
-        SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
-        sol = benchmark_nlsolve_oop(newton_fails, u0, p; solver = alg)
-        @test SciMLBase.successful_retcode(sol)
-        @test all(abs.(newton_fails(sol.u, p)) .< 1e-9)
-    end
-end
-
-# --- Allocation Checks ---
-
-## SimpleDFSane needs to allocate a history vector
-@testcase "Allocation Checks: $(_nameof(alg))" for alg in (SimpleNewtonRaphson(),
-    SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
-    SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
-    SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)),
-    SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
-    @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
-
-    nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
-    nlprob_sa = NonlinearProblem{false}(quadratic_f, @SVector[1.0, 1.0], 2.0)
-
-    try
-        nlsolve(nlprob_scalar, alg)
-        @test true
-    catch e
-        @error e
-        @test false
-    end
-
-    # ForwardDiff allocates for hessian since we don't propagate the chunksize
-    try
-        nlsolve(nlprob_sa, alg)
-        @test true
-    catch e
-        @error e
-        @test false broken=(alg isa SimpleHalley)
-    end
-end
-
-# --- Interval Nonlinear Problems ---
-
-@testcase "Interval Nonlinear Problem: $(alg)" for alg in (Bisection(), Falsi(), Ridder(),
-    Brent(), ITP(), Alefeld())
-    tspan = (1.0, 20.0)
-
-    function g(p)
-        probN = IntervalNonlinearProblem{false}(quadratic_f, typeof(p).(tspan), p)
-        sol = solve(probN, alg; abstol = 1e-9)
-        return sol.left
-    end
-
-    for p in 1.1:0.1:100.0
-        @test g(p)≈sqrt(p) atol=1e-3 rtol=1e-3
-        @test ForwardDiff.derivative(g, p)≈1 / (2 * sqrt(p)) atol=1e-3 rtol=1e-3
-    end
-
-    t = (p) -> [sqrt(p[2] / p[1])]
-    p = [0.9, 50.0]
-
-    function g2(p)
-        probN = IntervalNonlinearProblem{false}((u, p) -> p[1] * u * u - p[2], tspan, p)
-        sol = solve(probN, alg; abstol = 1e-9)
-        return [sol.u]
-    end
-
-    @test g2(p)≈[sqrt(p[2] / p[1])] atol=1e-3 rtol=1e-3
-    @test ForwardDiff.jacobian(g2, p)≈ForwardDiff.jacobian(t, p) atol=1e-3 rtol=1e-3
-
-    probB = IntervalNonlinearProblem{false}(quadratic_f, (1.0, 2.0), 2.0)
-    sol = solve(probB, alg; abstol = 1e-9)
-    @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
-
-    if !(alg isa Bisection || alg isa Falsi)
-        probB = IntervalNonlinearProblem{false}(quadratic_f, (sqrt(2.0), 10.0), 2.0)
-        sol = solve(probB, alg; abstol = 1e-9)
-        @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
-
-        probB = IntervalNonlinearProblem{false}(quadratic_f, (0.0, sqrt(2.0)), 2.0)
-        sol = solve(probB, alg; abstol = 1e-9)
-        @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
-    end
-end
-
-@testcase "Tolerance Tests Interval Methods: $(alg)" for alg in (Bisection(), Falsi(),
-    ITP())
-    tspan = (1.0, 20.0)
-    probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
-    tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
-    ϵ = eps(1.0) #least possible tol for all methods
-
-    for atol in tols
-        sol = solve(probB, alg; abstol = atol)
-        @test abs(sol.u - sqrt(2)) < atol
-        @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
-    end
-end
-
-@testcase "Tolerance Tests Interval Methods: $(alg)" for alg in (Ridder(), Brent())
-    tspan = (1.0, 20.0)
-    probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
-    tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution
-    ϵ = eps(1.0) #least possible tol for all methods
-
-    for atol in tols
-        sol = solve(probB, alg; abstol = atol)
-        @test abs(sol.u - sqrt(2)) < atol
-        @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
-    end
-end
-
-@testcase "Flipped Signs and Reversed Tspan: $(alg)" for alg in (Alefeld(), Bisection(),
-    Falsi(), Brent(), ITP(), Ridder())
-    f1(u, p) = u * u - p
-    f2(u, p) = p - u * u
-
-    for p in 1:4
-        inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
-        inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
-        inp3 = IntervalNonlinearProblem(f1, (2.0, 1.0), p)
-        inp4 = IntervalNonlinearProblem(f2, (2.0, 1.0), p)
-        @test abs.(solve(inp1, alg).u) ≈ sqrt.(p)
-        @test abs.(solve(inp2, alg).u) ≈ sqrt.(p)
-        @test abs.(solve(inp3, alg).u) ≈ sqrt.(p)
-        @test abs.(solve(inp4, alg).u) ≈ sqrt.(p)
-    end
-end
-
-# The following tests were included in the previos versions but these kwargs never did
-# anything!
-# # Garuntee Tests for Bisection
-# f = function (u, p)
-#     if u < 2.0
-#         return u - 2.0
-#     elseif u > 3.0
-#         return u - 3.0
-#     else
-#         return 0.0
-#     end
-# end
-# probB = IntervalNonlinearProblem(f, (0.0, 4.0))
-
-# sol = solve(probB, Bisection(; exact_left = true))
-# @test f(sol.left, nothing) < 0.0
-# @test f(nextfloat(sol.left), nothing) >= 0.0
-
-# sol = solve(probB, Bisection(; exact_right = true))
-# @test f(sol.right, nothing) >= 0.0
-# @test f(prevfloat(sol.right), nothing) <= 0.0
-
-# sol = solve(probB, Bisection(; exact_left = true, exact_right = true); immutable = false)
-# @test f(sol.left, nothing) < 0.0
-# @test f(nextfloat(sol.left), nothing) >= 0.0
-# @test f(sol.right, nothing) >= 0.0
-# @test f(prevfloat(sol.right), nothing) <= 0.0
diff --git a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
new file mode 100644
index 000000000..4e5401a02
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
@@ -0,0 +1,89 @@
+@testsetup module RobustnessTesting
+using LinearAlgebra, NonlinearProblemLibrary, DiffEqBase, Test
+
+problems = NonlinearProblemLibrary.problems
+dicts = NonlinearProblemLibrary.dicts
+
+function test_on_library(problems, dicts, alg_ops, broken_tests, ϵ = 1e-4;
+        skip_tests = nothing)
+    for (idx, (problem, dict)) in enumerate(zip(problems, dicts))
+        x = dict["start"]
+        res = similar(x)
+        nlprob = NonlinearProblem(problem, copy(x))
+        @testset "$idx: $(dict["title"])" begin
+            for alg in alg_ops
+                try
+                    sol = solve(nlprob, alg;
+                        termination_condition = AbsNormTerminationMode())
+                    problem(res, sol.u, nothing)
+
+                    skip = skip_tests !== nothing && idx in skip_tests[alg]
+                    if skip
+                        @test_skip norm(res) ≤ ϵ
+                        continue
+                    end
+                    broken = idx in broken_tests[alg] ? true : false
+                    @test norm(res)≤ϵ broken=broken
+                catch e
+                    @error e
+                    broken = idx in broken_tests[alg] ? true : false
+                    if broken
+                        @test false broken=true
+                    else
+                        @test 1 == 2
+                    end
+                end
+            end
+        end
+    end
+end
+
+export problems, dicts, test_on_library
+end
+
+@testitem "SimpleNewtonRaphson" setup=[RobustnessTesting] begin
+    alg_ops = (SimpleNewtonRaphson(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = []
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "SimpleTrustRegion" setup=[RobustnessTesting] begin
+    alg_ops = (SimpleTrustRegion(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [3, 15, 16, 21]
+    broken_tests[alg_ops[2]] = [15, 16]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "SimpleDFSane" setup=[RobustnessTesting] begin
+    alg_ops = (SimpleDFSane(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 11, 21]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "SimpleBroyden" setup=[RobustnessTesting] begin
+    alg_ops = (SimpleBroyden(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [1, 5, 11]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "SimpleKlement" setup=[RobustnessTesting] begin
+    alg_ops = (SimpleKlement(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 11, 12, 22]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
diff --git a/lib/SimpleNonlinearSolve/test/adjoint.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
similarity index 73%
rename from lib/SimpleNonlinearSolve/test/adjoint.jl
rename to lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
index cf2ef38d7..f96491449 100644
--- a/lib/SimpleNonlinearSolve/test/adjoint.jl
+++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
@@ -1,6 +1,6 @@
-using ForwardDiff, SciMLSensitivity, SimpleNonlinearSolve, XUnit, Zygote
+@testitem "Simple Adjoint Test" begin
+    using ForwardDiff, SciMLSensitivity, Zygote
 
-@testcase "Simple Adjoint Test" begin
     ff(u, p) = u .^ 2 .- p
 
     function solve_nlprob(p)
diff --git a/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
new file mode 100644
index 000000000..e7fc44f02
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
@@ -0,0 +1,90 @@
+@testsetup module ForwardADTesting
+using Reexport
+@reexport using ForwardDiff, SimpleNonlinearSolve, StaticArrays, LinearAlgebra
+import SimpleNonlinearSolve: AbstractSimpleNonlinearSolveAlgorithm
+
+test_f!(du, u, p) = (@. du = u^2 - p)
+test_f(u, p) = (@. u^2 - p)
+
+jacobian_f(::Number, p) = 1 / (2 * √p)
+jacobian_f(::Number, p::Number) = 1 / (2 * √p)
+jacobian_f(u, p::Number) = one.(u) .* (1 / (2 * √p))
+jacobian_f(u, p::AbstractArray) = diagm(vec(@. 1 / (2 * √p)))
+
+function solve_with(::Val{mode}, u, alg) where {mode}
+    f = if mode === :iip
+        solve_iip(p) = solve(NonlinearProblem(test_f!, u, p), alg).u
+    elseif mode === :oop
+        solve_oop(p) = solve(NonlinearProblem(test_f, u, p), alg).u
+    end
+    return f
+end
+
+__compatible(::Any, ::Val{:oop}) = true
+__compatible(::Number, ::Val{:iip}) = false
+__compatible(::AbstractArray, ::Val{:iip}) = true
+__compatible(::StaticArray, ::Val{:iip}) = false
+
+__compatible(::Any, ::Number) = true
+__compatible(::Number, ::AbstractArray) = false
+__compatible(u::AbstractArray, p::AbstractArray) = size(u) == size(p)
+
+__compatible(u::Number, ::AbstractSimpleNonlinearSolveAlgorithm) = true
+__compatible(u::AbstractArray, ::AbstractSimpleNonlinearSolveAlgorithm) = true
+__compatible(u::StaticArray, ::AbstractSimpleNonlinearSolveAlgorithm) = true
+
+__compatible(::AbstractSimpleNonlinearSolveAlgorithm, ::Val{:iip}) = true
+__compatible(::AbstractSimpleNonlinearSolveAlgorithm, ::Val{:oop}) = true
+__compatible(::SimpleHalley, ::Val{:iip}) = false
+
+export test_f, test_f!, jacobian_f, solve_with, __compatible
+end
+
+@testitem "ForwardDiff.jl Integration" setup=[ForwardADTesting] begin
+    @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
+        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+        SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleDFSane())
+        us = (2.0, @SVector[1.0, 1.0], [1.0, 1.0], ones(2, 2), @SArray ones(2, 2))
+
+        @testset "Scalar AD" begin
+            for p in 1.0:0.1:100.0, u0 in us, mode in (:iip, :oop)
+                __compatible(u0, alg) || continue
+                __compatible(u0, Val(mode)) || continue
+                __compatible(alg, Val(mode)) || continue
+
+                sol = solve(NonlinearProblem(test_f, u0, p), alg)
+                if SciMLBase.successful_retcode(sol)
+                    gs = abs.(ForwardDiff.derivative(solve_with(Val{mode}(), u0, alg), p))
+                    gs_true = abs.(jacobian_f(u0, p))
+                    if !(isapprox(gs, gs_true, atol = 1e-5))
+                        @show sol.retcode, sol.u
+                        @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_gradient=gs true_gradient=gs_true
+                    else
+                        @test abs.(gs)≈abs.(gs_true) atol=1e-5
+                    end
+                end
+            end
+        end
+
+        @testset "Jacobian" begin
+            for u0 in us, p in ([2.0, 1.0], [2.0 1.0; 3.0 4.0]), mode in (:iip, :oop)
+                __compatible(u0, p) || continue
+                __compatible(u0, alg) || continue
+                __compatible(u0, Val(mode)) || continue
+                __compatible(alg, Val(mode)) || continue
+
+                sol = solve(NonlinearProblem(test_f, u0, p), alg)
+                if SciMLBase.successful_retcode(sol)
+                    gs = abs.(ForwardDiff.jacobian(solve_with(Val{mode}(), u0, alg), p))
+                    gs_true = abs.(jacobian_f(u0, p))
+                    if !(isapprox(gs, gs_true, atol = 1e-5))
+                        @show sol.retcode, sol.u
+                        @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_jacobian=gs true_jacobian=gs_true
+                    else
+                        @test abs.(gs)≈abs.(gs_true) atol=1e-5
+                    end
+                end
+            end
+        end
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl b/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
new file mode 100644
index 000000000..8d99d3544
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
@@ -0,0 +1,24 @@
+@testitem "Nonlinear Least Squares" begin
+    using LinearAlgebra
+
+    true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
+
+    θ_true = [1.0, 0.1, 2.0, 0.5]
+    x = [-1.0, -0.5, 0.0, 0.5, 1.0]
+    y_target = true_function(x, θ_true)
+
+    function loss_function(θ, p)
+        ŷ = true_function(p, θ)
+        return ŷ .- y_target
+    end
+
+    θ_init = θ_true .+ 0.1
+    prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
+
+    @testset "Solver: $(nameof(typeof(solver)))" for solver in [
+        SimpleNewtonRaphson(AutoForwardDiff()), SimpleGaussNewton(AutoForwardDiff()),
+        SimpleNewtonRaphson(AutoFiniteDiff()), SimpleGaussNewton(AutoFiniteDiff())]
+        sol = solve(prob_oop, solver)
+        @test norm(sol.resid, Inf) < 1e-12
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
new file mode 100644
index 000000000..54cf86b21
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
@@ -0,0 +1,14 @@
+@testitem "Matrix Resizing" begin
+    ff(u, p) = u .* u .- p
+    u0 = ones(2, 3)
+    p = 2.0
+    vecprob = NonlinearProblem(ff, vec(u0), p)
+    prob = NonlinearProblem(ff, u0, p)
+
+    @testset "$(nameof(typeof(alg)))" for alg in (SimpleKlement(), SimpleBroyden(),
+        SimpleNewtonRaphson(), SimpleDFSane(),
+        SimpleLimitedMemoryBroyden(; threshold = Val(2)),
+        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
+        @test vec(solve(prob, alg).u) ≈ solve(vecprob, alg).u
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
new file mode 100644
index 000000000..dc6054273
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -0,0 +1,279 @@
+@testsetup module RootfindingTesting
+using Reexport
+@reexport using AllocCheck,
+    LinearSolve, StaticArrays, Random, LinearAlgebra, ForwardDiff, DiffEqBase
+import PolyesterForwardDiff
+
+quadratic_f(u, p) = u .* u .- p
+quadratic_f!(du, u, p) = (du .= u .* u .- p)
+quadratic_f2(u, p) = @. p[1] * u * u - p[2]
+
+function newton_fails(u, p)
+    return 0.010000000000000002 .+
+           10.000000000000002 ./ (1 .+
+            (0.21640425613334457 .+
+             216.40425613334457 ./ (1 .+
+              (0.21640425613334457 .+
+               216.40425613334457 ./
+               (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
+           0.0011552453009332421u .- p
+end
+
+const TERMINATION_CONDITIONS = [
+    NormTerminationMode(), RelTerminationMode(), RelNormTerminationMode(),
+    AbsTerminationMode(), AbsNormTerminationMode(), RelSafeTerminationMode(),
+    AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode(),
+]
+
+function benchmark_nlsolve_oop(f::F, u0, p = 2.0; solver) where {F}
+    prob = NonlinearProblem{false}(f, u0, p)
+    return solve(prob, solver; abstol = 1e-9)
+end
+function benchmark_nlsolve_iip(f!::F, u0, p = 2.0; solver) where {F}
+    prob = NonlinearProblem{true}(f!, u0, p)
+    return solve(prob, solver; abstol = 1e-9)
+end
+
+export quadratic_f, quadratic_f!, quadratic_f2, newton_fails, TERMINATION_CONDITIONS,
+    benchmark_nlsolve_oop, benchmark_nlsolve_iip
+
+end
+
+@testitem "First Order Methods" setup=[RootfindingTesting] begin
+    @testset "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion,
+        (args...; kwargs...) -> SimpleTrustRegion(args...; nlsolve_update_rule = Val(true),
+            kwargs...))
+        @testset "AutoDiff: $(nameof(typeof(autodiff))))" for autodiff in (AutoFiniteDiff(),
+            AutoForwardDiff(), AutoPolyesterForwardDiff())
+            @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],
+                @SVector[1.0, 1.0], 1.0)
+                u0 isa SVector && autodiff isa AutoPolyesterForwardDiff && continue
+                sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = alg(; autodiff))
+                @test SciMLBase.successful_retcode(sol)
+                @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+            end
+
+            @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
+                sol = benchmark_nlsolve_iip(quadratic_f!, u0; solver = alg(; autodiff))
+                @test SciMLBase.successful_retcode(sol)
+                @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+            end
+        end
+
+        @testset "Termination condition: $(termination_condition) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+            u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
+
+            probN = NonlinearProblem(quadratic_f, u0, 2.0)
+            @test all(solve(probN, alg(); termination_condition).u .≈ sqrt(2.0))
+        end
+    end
+end
+
+@testitem "SimpleHalley" setup=[RootfindingTesting] begin
+    @testset "AutoDiff: $(nameof(typeof(autodiff)))" for autodiff in (AutoFiniteDiff(),
+        AutoForwardDiff())
+        @testset "[OOP] u0: $(nameof(typeof(u0)))" for u0 in ([1.0, 1.0],
+            @SVector[1.0, 1.0], 1.0)
+            sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = SimpleHalley(; autodiff))
+            @test SciMLBase.successful_retcode(sol)
+            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+        end
+    end
+
+    @testset "Termination condition: $(termination_condition) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+        u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
+
+        probN = NonlinearProblem(quadratic_f, u0, 2.0)
+        @test all(solve(probN, SimpleHalley(); termination_condition).u .≈ sqrt(2.0))
+    end
+end
+
+@testitem "Derivative Free Metods" setup=[RootfindingTesting] begin
+    @testset "$(nameof(typeof(alg)))" for alg in [SimpleBroyden(), SimpleKlement(),
+        SimpleDFSane(), SimpleLimitedMemoryBroyden(),
+        SimpleBroyden(; linesearch = Val(true)),
+        SimpleLimitedMemoryBroyden(; linesearch = Val(true))]
+        @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
+            sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = alg)
+            @test SciMLBase.successful_retcode(sol)
+            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+        end
+
+        @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
+            sol = benchmark_nlsolve_iip(quadratic_f!, u0; solver = alg)
+            @test SciMLBase.successful_retcode(sol)
+            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+        end
+
+        @testset "Termination condition: $(termination_condition) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+            u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
+
+            probN = NonlinearProblem(quadratic_f, u0, 2.0)
+            @test all(solve(probN, alg; termination_condition).u .≈ sqrt(2.0))
+        end
+    end
+end
+
+@testitem "Newton Fails" setup=[RootfindingTesting] begin
+    u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
+    p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+
+    @testset "$(nameof(typeof(alg)))" for alg in (SimpleDFSane(), SimpleTrustRegion(),
+        SimpleHalley(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
+        sol = benchmark_nlsolve_oop(newton_fails, u0, p; solver = alg)
+        @test SciMLBase.successful_retcode(sol)
+        @test all(abs.(newton_fails(sol.u, p)) .< 1e-9)
+    end
+end
+
+@testitem "Allocation Checks" setup=[RootfindingTesting] begin
+    @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
+        SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
+        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+        SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)),
+        SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
+        @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
+
+        nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
+        nlprob_sa = NonlinearProblem{false}(quadratic_f, @SVector[1.0, 1.0], 2.0)
+
+        try
+            nlsolve(nlprob_scalar, alg)
+            @test true
+        catch e
+            @error e
+            @test false
+        end
+
+        # ForwardDiff allocates for hessian since we don't propagate the chunksize
+        try
+            nlsolve(nlprob_sa, alg)
+            @test true
+        catch e
+            @error e
+            @test false broken=(alg isa SimpleHalley)
+        end
+    end
+end
+
+@testitem "Interval Nonlinear Problems" setup=[RootfindingTesting] begin
+    @testset "$(nameof(typeof(alg)))" for alg in (Bisection(), Falsi(), Ridder(), Brent(),
+        ITP(), Alefeld())
+        tspan = (1.0, 20.0)
+
+        function g(p)
+            probN = IntervalNonlinearProblem{false}(quadratic_f, typeof(p).(tspan), p)
+            sol = solve(probN, alg; abstol = 1e-9)
+            return sol.left
+        end
+
+        for p in 1.1:0.1:100.0
+            @test g(p)≈sqrt(p) atol=1e-3 rtol=1e-3
+            @test ForwardDiff.derivative(g, p)≈1 / (2 * sqrt(p)) atol=1e-3 rtol=1e-3
+        end
+
+        t = (p) -> [sqrt(p[2] / p[1])]
+        p = [0.9, 50.0]
+
+        function g2(p)
+            probN = IntervalNonlinearProblem{false}((u, p) -> p[1] * u * u - p[2], tspan, p)
+            sol = solve(probN, alg; abstol = 1e-9)
+            return [sol.u]
+        end
+
+        @test g2(p)≈[sqrt(p[2] / p[1])] atol=1e-3 rtol=1e-3
+        @test ForwardDiff.jacobian(g2, p)≈ForwardDiff.jacobian(t, p) atol=1e-3 rtol=1e-3
+
+        probB = IntervalNonlinearProblem{false}(quadratic_f, (1.0, 2.0), 2.0)
+        sol = solve(probB, alg; abstol = 1e-9)
+        @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
+
+        if !(alg isa Bisection || alg isa Falsi)
+            probB = IntervalNonlinearProblem{false}(quadratic_f, (sqrt(2.0), 10.0), 2.0)
+            sol = solve(probB, alg; abstol = 1e-9)
+            @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
+
+            probB = IntervalNonlinearProblem{false}(quadratic_f, (0.0, sqrt(2.0)), 2.0)
+            sol = solve(probB, alg; abstol = 1e-9)
+            @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
+        end
+    end
+end
+
+@testitem "Tolerance Tests Interval Methods" setup=[RootfindingTesting] begin
+    @testset "$(nameof(typeof(alg)))" for alg in (Bisection(), Falsi(), ITP())
+        tspan = (1.0, 20.0)
+        probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
+        tols = [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
+        ϵ = eps(1.0) #least possible tol for all methods
+
+        for atol in tols
+            sol = solve(probB, alg; abstol = atol)
+            @test abs(sol.u - sqrt(2)) < atol
+            @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
+        end
+    end
+end
+
+@testitem "Tolerance Tests Interval Methods 2" setup=[RootfindingTesting] begin
+    @testset "$(nameof(typeof(alg)))" for alg in (Ridder(), Brent())
+        tspan = (1.0, 20.0)
+        probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
+        tols = [0.1] # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it converges with max precision to the solution
+        ϵ = eps(1.0) #least possible tol for all methods
+
+        for atol in tols
+            sol = solve(probB, alg; abstol = atol)
+            @test abs(sol.u - sqrt(2)) < atol
+            @test abs(sol.u - sqrt(2)) > ϵ #test that the solution is not calculated upto max precision
+        end
+    end
+end
+
+@testitem "Flipped Signs and Reversed Tspan" setup=[RootfindingTesting] begin
+    @testset "$(nameof(typeof(alg)))" for alg in (Alefeld(), Bisection(), Falsi(), Brent(),
+        ITP(), Ridder())
+        f1(u, p) = u * u - p
+        f2(u, p) = p - u * u
+
+        for p in 1:4
+            inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
+            inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
+            inp3 = IntervalNonlinearProblem(f1, (2.0, 1.0), p)
+            inp4 = IntervalNonlinearProblem(f2, (2.0, 1.0), p)
+            @test abs.(solve(inp1, alg).u) ≈ sqrt.(p)
+            @test abs.(solve(inp2, alg).u) ≈ sqrt.(p)
+            @test abs.(solve(inp3, alg).u) ≈ sqrt.(p)
+            @test abs.(solve(inp4, alg).u) ≈ sqrt.(p)
+        end
+    end
+end
+
+# The following tests were included in the previos versions but these kwargs never did
+# anything!
+# # Garuntee Tests for Bisection
+# f = function (u, p)
+#     if u < 2.0
+#         return u - 2.0
+#     elseif u > 3.0
+#         return u - 3.0
+#     else
+#         return 0.0
+#     end
+# end
+# probB = IntervalNonlinearProblem(f, (0.0, 4.0))
+
+# sol = solve(probB, Bisection(; exact_left = true))
+# @test f(sol.left, nothing) < 0.0
+# @test f(nextfloat(sol.left), nothing) >= 0.0
+
+# sol = solve(probB, Bisection(; exact_right = true))
+# @test f(sol.right, nothing) >= 0.0
+# @test f(prevfloat(sol.right), nothing) <= 0.0
+
+# sol = solve(probB, Bisection(; exact_left = true, exact_right = true); immutable = false)
+# @test f(sol.left, nothing) < 0.0
+# @test f(nextfloat(sol.left), nothing) >= 0.0
+# @test f(sol.right, nothing) >= 0.0
+# @test f(prevfloat(sol.right), nothing) <= 0.0
diff --git a/lib/SimpleNonlinearSolve/test/cuda/Project.toml b/lib/SimpleNonlinearSolve/test/cuda/Project.toml
deleted file mode 100644
index 4fdee40a7..000000000
--- a/lib/SimpleNonlinearSolve/test/cuda/Project.toml
+++ /dev/null
@@ -1,6 +0,0 @@
-[deps]
-CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
-SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-XUnit = "3e3c03f2-1a94-11e9-2981-050a4ca824ab"
diff --git a/lib/SimpleNonlinearSolve/test/forward_ad.jl b/lib/SimpleNonlinearSolve/test/forward_ad.jl
deleted file mode 100644
index 99b32c3cb..000000000
--- a/lib/SimpleNonlinearSolve/test/forward_ad.jl
+++ /dev/null
@@ -1,83 +0,0 @@
-using ForwardDiff, SimpleNonlinearSolve, StaticArrays, XUnit, LinearAlgebra
-import SimpleNonlinearSolve: AbstractSimpleNonlinearSolveAlgorithm
-
-test_f!(du, u, p) = (@. du = u^2 - p)
-test_f(u, p) = (@. u^2 - p)
-
-jacobian_f(::Number, p) = 1 / (2 * √p)
-jacobian_f(::Number, p::Number) = 1 / (2 * √p)
-jacobian_f(u, p::Number) = one.(u) .* (1 / (2 * √p))
-jacobian_f(u, p::AbstractArray) = diagm(vec(@. 1 / (2 * √p)))
-
-function solve_with(::Val{mode}, u, alg) where {mode}
-    f = if mode === :iip
-        solve_iip(p) = solve(NonlinearProblem(test_f!, u, p), alg).u
-    elseif mode === :oop
-        solve_oop(p) = solve(NonlinearProblem(test_f, u, p), alg).u
-    end
-    return f
-end
-
-__compatible(::Any, ::Val{:oop}) = true
-__compatible(::Number, ::Val{:iip}) = false
-__compatible(::AbstractArray, ::Val{:iip}) = true
-__compatible(::StaticArray, ::Val{:iip}) = false
-
-__compatible(::Any, ::Number) = true
-__compatible(::Number, ::AbstractArray) = false
-__compatible(u::AbstractArray, p::AbstractArray) = size(u) == size(p)
-
-__compatible(u::Number, ::AbstractSimpleNonlinearSolveAlgorithm) = true
-__compatible(u::AbstractArray, ::AbstractSimpleNonlinearSolveAlgorithm) = true
-__compatible(u::StaticArray, ::AbstractSimpleNonlinearSolveAlgorithm) = true
-
-__compatible(::AbstractSimpleNonlinearSolveAlgorithm, ::Val{:iip}) = true
-__compatible(::AbstractSimpleNonlinearSolveAlgorithm, ::Val{:oop}) = true
-__compatible(::SimpleHalley, ::Val{:iip}) = false
-
-@testcase "ForwardDiff.jl Integration: $(alg)" for alg in (SimpleNewtonRaphson(),
-    SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
-    SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleDFSane())
-    us = (2.0, @SVector[1.0, 1.0], [1.0, 1.0], ones(2, 2), @SArray ones(2, 2))
-
-    @testset "Scalar AD" begin
-        for p in 1.0:0.1:100.0, u0 in us, mode in (:iip, :oop)
-            __compatible(u0, alg) || continue
-            __compatible(u0, Val(mode)) || continue
-            __compatible(alg, Val(mode)) || continue
-
-            sol = solve(NonlinearProblem(test_f, u0, p), alg)
-            if SciMLBase.successful_retcode(sol)
-                gs = abs.(ForwardDiff.derivative(solve_with(Val{mode}(), u0, alg), p))
-                gs_true = abs.(jacobian_f(u0, p))
-                if !(isapprox(gs, gs_true, atol = 1e-5))
-                    @show sol.retcode, sol.u
-                    @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_gradient=gs true_gradient=gs_true
-                else
-                    @test abs.(gs)≈abs.(gs_true) atol=1e-5
-                end
-            end
-        end
-    end
-
-    @testset "Jacobian" begin
-        for u0 in us, p in ([2.0, 1.0], [2.0 1.0; 3.0 4.0]), mode in (:iip, :oop)
-            __compatible(u0, p) || continue
-            __compatible(u0, alg) || continue
-            __compatible(u0, Val(mode)) || continue
-            __compatible(alg, Val(mode)) || continue
-
-            sol = solve(NonlinearProblem(test_f, u0, p), alg)
-            if SciMLBase.successful_retcode(sol)
-                gs = abs.(ForwardDiff.jacobian(solve_with(Val{mode}(), u0, alg), p))
-                gs_true = abs.(jacobian_f(u0, p))
-                if !(isapprox(gs, gs_true, atol = 1e-5))
-                    @show sol.retcode, sol.u
-                    @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_jacobian=gs true_jacobian=gs_true
-                else
-                    @test abs.(gs)≈abs.(gs_true) atol=1e-5
-                end
-            end
-        end
-    end
-end
diff --git a/lib/SimpleNonlinearSolve/test/cuda.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
similarity index 75%
rename from lib/SimpleNonlinearSolve/test/cuda.jl
rename to lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
index f1f372f6a..37999dada 100644
--- a/lib/SimpleNonlinearSolve/test/cuda.jl
+++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
@@ -1,12 +1,12 @@
-using SimpleNonlinearSolve, StaticArrays, CUDA, XUnit
+@testitem "Solving on GPUs" begin
+    using StaticArrays, CUDA
 
-CUDA.allowscalar(false)
+    CUDA.allowscalar(false)
 
-f(u, p) = u .* u .- 2
-f!(du, u, p) = du .= u .* u .- 2
+    f(u, p) = u .* u .- 2
+    f!(du, u, p) = du .= u .* u .- 2
 
-@testset "Solving on GPUs" begin
-    @testcase "$(alg)" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
+    @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
         SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
         SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
         SimpleHalley(), SimpleBroyden(; linesearch = Val(true)),
@@ -36,15 +36,22 @@ f!(du, u, p) = du .= u .* u .- 2
     end
 end
 
-function kernel_function(prob, alg)
-    solve(prob, alg)
-    return nothing
-end
+@testitem "CUDA Kernel Launch Test" begin
+    using StaticArrays, CUDA
+
+    CUDA.allowscalar(false)
+
+    f(u, p) = u .* u .- 2
+    f!(du, u, p) = du .= u .* u .- 2
+
+    function kernel_function(prob, alg)
+        solve(prob, alg)
+        return nothing
+    end
 
-@testset "CUDA Kernel Launch Test" begin
     prob = NonlinearProblem{false}(f, @SVector[1.0f0, 1.0f0])
 
-    @testcase "$(alg)" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
+    @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
         SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
         SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
         SimpleBroyden(; linesearch = Val(true)),
diff --git a/lib/SimpleNonlinearSolve/test/least_squares.jl b/lib/SimpleNonlinearSolve/test/least_squares.jl
deleted file mode 100644
index 6c48a9807..000000000
--- a/lib/SimpleNonlinearSolve/test/least_squares.jl
+++ /dev/null
@@ -1,22 +0,0 @@
-using SimpleNonlinearSolve, LinearAlgebra, XUnit
-
-true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
-
-θ_true = [1.0, 0.1, 2.0, 0.5]
-x = [-1.0, -0.5, 0.0, 0.5, 1.0]
-y_target = true_function(x, θ_true)
-
-function loss_function(θ, p)
-    ŷ = true_function(p, θ)
-    return ŷ .- y_target
-end
-
-θ_init = θ_true .+ 0.1
-prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
-
-@testcase "Solver: $(solver)" for solver in [SimpleNewtonRaphson(AutoForwardDiff()),
-    SimpleGaussNewton(AutoForwardDiff()),
-    SimpleNewtonRaphson(AutoFiniteDiff()), SimpleGaussNewton(AutoFiniteDiff())]
-    sol = solve(prob_oop, solver)
-    @test norm(sol.resid) < 1e-12
-end
diff --git a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
deleted file mode 100644
index 778ec5185..000000000
--- a/lib/SimpleNonlinearSolve/test/matrix_resizing_tests.jl
+++ /dev/null
@@ -1,13 +0,0 @@
-using SimpleNonlinearSolve, XUnit
-
-ff(u, p) = u .* u .- p
-u0 = ones(2, 3)
-p = 2.0
-vecprob = NonlinearProblem(ff, vec(u0), p)
-prob = NonlinearProblem(ff, u0, p)
-
-@testcase "$(alg)" for alg in (SimpleKlement(), SimpleBroyden(), SimpleNewtonRaphson(),
-    SimpleDFSane(), SimpleLimitedMemoryBroyden(; threshold = Val(2)), SimpleTrustRegion(),
-    SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
-    @test vec(solve(prob, alg).u) ≈ solve(vecprob, alg).u
-end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 5d9216ae3..56acdd0c5 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -1,25 +1,11 @@
-using Pkg, SafeTestsets, XUnit
+using ReTestItems
 
 const GROUP = get(ENV, "GROUP", "All")
 
-function activate_env(env)
-    Pkg.activate(env)
-    Pkg.develop(PackageSpec(path = dirname(@__DIR__)))
-    Pkg.instantiate()
+if GROUP == "All" || GROUP == "Core"
+    ReTestItems.runtests(joinpath(@__DIR__, "core/"))
 end
 
-@testset runner=ParallelTestRunner() "SimpleNonlinearSolve.jl" begin
-    if GROUP == "All" || GROUP == "Core"
-        @safetestset "Basic Tests" include("basictests.jl")
-        @safetestset "Forward AD" include("forward_ad.jl")
-        @safetestset "Matrix Resizing Tests" include("matrix_resizing_tests.jl")
-        @safetestset "Least Squares Tests" include("least_squares.jl")
-        @safetestset "23 Test Problems" include("23_test_problems.jl")
-        @safetestset "Simple Adjoint Tests" include("adjoint.jl")
-    end
-
-    if GROUP == "CUDA"
-        activate_env("cuda")
-        @safetestset "CUDA Tests" include("cuda.jl")
-    end
-end
+if GROUP == "GPU"
+    ReTestItems.runtests(joinpath(@__DIR__, "gpu/"))
+end
\ No newline at end of file

From 5fd1cd7bccbbcc720c83ad8cb748f43667c64011 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 9 Feb 2024 09:47:39 -0500
Subject: [PATCH 304/375] Test on multiple os

---
 lib/SimpleNonlinearSolve/.github/workflows/CI.yml | 13 +++++++++++--
 lib/SimpleNonlinearSolve/test/runtests.jl         |  2 +-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index 009bccfbf..8e0ab3bff 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -8,13 +8,17 @@ on:
       - main
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         group:
           - Core
         version:
           - '1'
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
@@ -40,6 +44,11 @@ jobs:
           RETESTITEMS_NWORKERS: 4
           RETESTITEMS_NWORKER_THREADS: 2
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v3
+        with:
+          directories: src,ext
+      - uses: codecov/codecov-action@v4
         with:
           file: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true
+          fail_ci_if_error: true
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 56acdd0c5..1a2e93bb1 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -8,4 +8,4 @@ end
 
 if GROUP == "GPU"
     ReTestItems.runtests(joinpath(@__DIR__, "gpu/"))
-end
\ No newline at end of file
+end

From 75911566db61ee80a8dc39938b3000a654bb0e03 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 9 Feb 2024 10:13:46 -0500
Subject: [PATCH 305/375] Retry for Broyden

---
 lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
index 4e5401a02..8b8f2395d 100644
--- a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
@@ -70,7 +70,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "SimpleBroyden" setup=[RobustnessTesting] begin
+@testitem "SimpleBroyden" retries=5 setup=[RobustnessTesting] begin
     alg_ops = (SimpleBroyden(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)

From 2d216726aa1a0201ad3a4986676f22f9c44bbd2c Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 10 Feb 2024 12:02:07 -0500
Subject: [PATCH 306/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index b04fc411c..ca54b2515 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -32,7 +32,7 @@ SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
 ADTypes = "0.2.6"
 ArrayInterface = "7.7"
 ChainRulesCore = "1.21"
-ConcreteStructs = "0.2.2"
+ConcreteStructs = "0.2.4"
 DiffEqBase = "6.146"
 FastClosures = "0.3"
 FiniteDiff = "2.22"

From ef11257832fb05d529c58065296450ab24779c13 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 10 Feb 2024 12:30:33 -0500
Subject: [PATCH 307/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index ca54b2515..bf943823d 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -32,7 +32,7 @@ SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
 ADTypes = "0.2.6"
 ArrayInterface = "7.7"
 ChainRulesCore = "1.21"
-ConcreteStructs = "0.2.4"
+ConcreteStructs = "0.2.3"
 DiffEqBase = "6.146"
 FastClosures = "0.3"
 FiniteDiff = "2.22"

From 4d1c76191b7c5e58b17be42cefe5bbb0a1f5a929 Mon Sep 17 00:00:00 2001
From: ChrisRackauckas <1814174+ChrisRackauckas@users.noreply.github.com>
Date: Sun, 11 Feb 2024 00:19:29 +0000
Subject: [PATCH 308/375] Format .jl files

---
 .../src/SimpleNonlinearSolve.jl                 | 17 ++++++++++-------
 lib/SimpleNonlinearSolve/src/ad.jl              |  9 ++++++---
 .../src/bracketing/alefeld.jl                   |  9 ++++++---
 .../src/bracketing/bisection.jl                 |  3 ++-
 .../src/bracketing/brent.jl                     |  3 ++-
 .../src/bracketing/falsi.jl                     |  3 ++-
 lib/SimpleNonlinearSolve/src/bracketing/itp.jl  |  3 ++-
 .../src/bracketing/ridder.jl                    |  3 ++-
 .../src/nlsolve/lbroyden.jl                     |  3 ++-
 .../test/core/rootfind_tests.jl                 |  9 +++++----
 10 files changed, 39 insertions(+), 23 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 3f12d8888..894195106 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -4,12 +4,13 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
 
 @recompile_invalidations begin
     using ADTypes, ArrayInterface, ConcreteStructs, DiffEqBase, FastClosures, FiniteDiff,
-        ForwardDiff, Reexport, LinearAlgebra, SciMLBase
+          ForwardDiff, Reexport, LinearAlgebra, SciMLBase
 
     import DiffEqBase: AbstractNonlinearTerminationMode,
-        AbstractSafeNonlinearTerminationMode, AbstractSafeBestNonlinearTerminationMode,
-        NonlinearSafeTerminationReturnCode, get_termination_mode,
-        NONLINEARSOLVE_DEFAULT_NORM
+                       AbstractSafeNonlinearTerminationMode,
+                       AbstractSafeBestNonlinearTerminationMode,
+                       NonlinearSafeTerminationReturnCode, get_termination_mode,
+                       NONLINEARSOLVE_DEFAULT_NORM
     import ForwardDiff: Dual
     import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
     import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace, _unwrap_val
@@ -56,14 +57,16 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing, args...;
 end
 
 # By Pass the highlevel checks for NonlinearProblem for Simple Algorithms
-function SciMLBase.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
+function SciMLBase.solve(
+        prob::NonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
         args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...)
     if sensealg === nothing && haskey(prob.kwargs, :sensealg)
         sensealg = prob.kwargs[:sensealg]
     end
     new_u0 = u0 !== nothing ? u0 : prob.u0
     new_p = p !== nothing ? p : prob.p
-    return __internal_solve_up(prob, sensealg, new_u0, u0 === nothing, new_p, p === nothing,
+    return __internal_solve_up(
+        prob, sensealg, new_u0, u0 === nothing, new_p, p === nothing,
         alg, args...; kwargs...)
 end
 
@@ -111,7 +114,7 @@ end
 end
 
 export SimpleBroyden, SimpleDFSane, SimpleGaussNewton, SimpleHalley, SimpleKlement,
-    SimpleLimitedMemoryBroyden, SimpleNewtonRaphson, SimpleTrustRegion
+       SimpleLimitedMemoryBroyden, SimpleNewtonRaphson, SimpleTrustRegion
 export Alefeld, Bisection, Brent, Falsi, ITP, Ridder
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index a4a777be6..b4f7f41b9 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -1,15 +1,18 @@
-function SciMLBase.solve(prob::NonlinearProblem{<:Union{Number, <:AbstractArray},
+function SciMLBase.solve(
+        prob::NonlinearProblem{<:Union{Number, <:AbstractArray},
             iip, <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
         alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) where {T, V, P, iip}
     sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
     dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
-    return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats,
+    return SciMLBase.build_solution(
+        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats,
         sol.original)
 end
 
 for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
     @eval begin
-        function SciMLBase.solve(prob::IntervalNonlinearProblem{uType, iip,
+        function SciMLBase.solve(
+                prob::IntervalNonlinearProblem{uType, iip,
                     <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
                 alg::$(algType), args...; kwargs...) where {uType, T, V, P, iip}
             sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl b/lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl
index 39c984fd5..3b89751a7 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl
@@ -38,7 +38,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
         end
         ē, fc = d, f(c)
         (a == c || b == c) &&
-            return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+            return build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
                 left = a, right = b)
         iszero(fc) &&
             return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
@@ -57,7 +58,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
         end
         fc = f(c)
         (ā == c || b̄ == c) &&
-            return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+            return build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
                 left = ā, right = b̄)
         iszero(fc) &&
             return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
@@ -76,7 +78,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
         end
         fc = f(c)
         (ā == c || b̄ == c) &&
-            return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+            return build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
                 left = ā, right = b̄)
         iszero(fc) &&
             return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
index 38f9cb9eb..acadf6aa1 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
@@ -35,7 +35,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
     end
 
     if iszero(fr)
-        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+        return build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
             left, right)
     end
 
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
index f37c45e4a..89b2e60be 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
@@ -22,7 +22,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
     end
 
     if iszero(fr)
-        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+        return build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
             left, right)
     end
 
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
index 86086f81a..896e07329 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
@@ -21,7 +21,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
     end
 
     if iszero(fr)
-        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+        return build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
             left, right)
     end
 
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index cce5eafea..3f2069b72 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -67,7 +67,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
     end
 
     if iszero(fr)
-        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+        return build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
             left, right)
     end
     ϵ = abstol
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
index cd18060d5..3b23f4287 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
@@ -21,7 +21,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
     end
 
     if iszero(fr)
-        return build_solution(prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
+        return build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
             left, right)
     end
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 9f8896ed1..145a5467d 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -137,7 +137,8 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
         init_α = inv(alg.alpha)
     end
 
-    converged, res = __unrolled_lbroyden_initial_iterations(prob, xo, fo, δx, abstol, U, Vᵀ,
+    converged, res = __unrolled_lbroyden_initial_iterations(
+        prob, xo, fo, δx, abstol, U, Vᵀ,
         threshold, ls_cache, init_α)
 
     converged &&
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index dc6054273..005651568 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -1,7 +1,7 @@
 @testsetup module RootfindingTesting
 using Reexport
 @reexport using AllocCheck,
-    LinearSolve, StaticArrays, Random, LinearAlgebra, ForwardDiff, DiffEqBase
+                LinearSolve, StaticArrays, Random, LinearAlgebra, ForwardDiff, DiffEqBase
 import PolyesterForwardDiff
 
 quadratic_f(u, p) = u .* u .- p
@@ -22,7 +22,7 @@ end
 const TERMINATION_CONDITIONS = [
     NormTerminationMode(), RelTerminationMode(), RelNormTerminationMode(),
     AbsTerminationMode(), AbsNormTerminationMode(), RelSafeTerminationMode(),
-    AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode(),
+    AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode()
 ]
 
 function benchmark_nlsolve_oop(f::F, u0, p = 2.0; solver) where {F}
@@ -35,7 +35,7 @@ function benchmark_nlsolve_iip(f!::F, u0, p = 2.0; solver) where {F}
 end
 
 export quadratic_f, quadratic_f!, quadratic_f2, newton_fails, TERMINATION_CONDITIONS,
-    benchmark_nlsolve_oop, benchmark_nlsolve_iip
+       benchmark_nlsolve_oop, benchmark_nlsolve_iip
 
 end
 
@@ -43,7 +43,8 @@ end
     @testset "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion,
         (args...; kwargs...) -> SimpleTrustRegion(args...; nlsolve_update_rule = Val(true),
             kwargs...))
-        @testset "AutoDiff: $(nameof(typeof(autodiff))))" for autodiff in (AutoFiniteDiff(),
+        @testset "AutoDiff: $(nameof(typeof(autodiff))))" for autodiff in (
+            AutoFiniteDiff(),
             AutoForwardDiff(), AutoPolyesterForwardDiff())
             @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],
                 @SVector[1.0, 1.0], 1.0)

From 214fb0fd8e33963dbbd1940919fbdd9aa732b25c Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 9 Feb 2024 14:58:18 -0500
Subject: [PATCH 309/375] Dispatch didnot propagate problem kwargs

---
 lib/SimpleNonlinearSolve/Project.toml                | 2 +-
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index bf943823d..fdef9b8ee 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.4.1"
+version = "1.4.2"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 894195106..55b579e95 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -53,7 +53,7 @@ include("ad.jl")
 # Set the default bracketing method to ITP
 SciMLBase.solve(prob::IntervalNonlinearProblem; kwargs...) = solve(prob, ITP(); kwargs...)
 function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing, args...; kwargs...)
-    return solve(prob, ITP(), args...; kwargs...)
+    return solve(prob, ITP(), args...; prob.kwargs..., kwargs...)
 end
 
 # By Pass the highlevel checks for NonlinearProblem for Simple Algorithms
@@ -67,7 +67,7 @@ function SciMLBase.solve(
     new_p = p !== nothing ? p : prob.p
     return __internal_solve_up(
         prob, sensealg, new_u0, u0 === nothing, new_p, p === nothing,
-        alg, args...; kwargs...)
+        alg, args...; prob.kwargs..., kwargs...)
 end
 
 function __internal_solve_up(_prob::NonlinearProblem, sensealg, u0, u0_changed, p,

From 3e8ffd0309a2af7030b117a99927ab91d7f3a00a Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 15 Feb 2024 11:06:29 -0500
Subject: [PATCH 310/375] Add a test for kwarg propagation

---
 lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index 005651568..f3ac188a9 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -120,14 +120,19 @@ end
     p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
 
     @testset "$(nameof(typeof(alg)))" for alg in (SimpleDFSane(), SimpleTrustRegion(),
-        SimpleHalley(),
-        SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
+        SimpleHalley(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
         sol = benchmark_nlsolve_oop(newton_fails, u0, p; solver = alg)
         @test SciMLBase.successful_retcode(sol)
         @test all(abs.(newton_fails(sol.u, p)) .< 1e-9)
     end
 end
 
+@testitem "Kwargs Propagation" setup=[RootfindingTesting] begin
+    prob = NonlinearProblem(quadratic_f, ones(4), 2.0; maxiters = 2)
+    sol = solve(prob, SimpleNewtonRaphson())
+    @test sol.retcode === ReturnCode.MaxIters
+end
+
 @testitem "Allocation Checks" setup=[RootfindingTesting] begin
     @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
         SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),

From 05c80bb151a6714abdbd6de991c3ef8af4da6a10 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 15 Feb 2024 11:40:00 -0500
Subject: [PATCH 311/375] Dont fail fast

---
 lib/SimpleNonlinearSolve/.github/workflows/CI.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index 8e0ab3bff..16ec1087c 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -10,6 +10,7 @@ jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         group:
           - Core

From 05c6206a9f447f977eab033511ecf12d185db58c Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 14 Feb 2024 22:59:34 -0500
Subject: [PATCH 312/375] Static GPU compilation of Jacobian

---
 .../.buildkite/pipeline.yml                     |  2 +-
 lib/SimpleNonlinearSolve/Project.toml           |  2 +-
 lib/SimpleNonlinearSolve/src/utils.jl           | 17 ++++++++++++++---
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
index ef04f99a6..f95a8351f 100644
--- a/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
+++ b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
@@ -13,7 +13,7 @@ steps:
     agents:
       queue: "juliagpu"
       cuda: "*"
-    timeout_in_minutes: 30
+    timeout_in_minutes: 120
     # Don't run Buildkite if the commit message includes the text [skip tests]
     if: build.message !~ /\[skip tests\]/
 
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index fdef9b8ee..4065b1fcf 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.4.2"
+version = "1.4.3"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 7390f5e3c..e89794b4c 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -40,17 +40,28 @@ function __pick_forwarddiff_chunk(x::StaticArray)
     end
 end
 
-function __get_jacobian_config(ad::AutoForwardDiff{CS}, f, x) where {CS}
+function __get_jacobian_config(ad::AutoForwardDiff{CS}, f::F, x) where {F, CS}
     ck = (CS === nothing || CS ≤ 0) ? __pick_forwarddiff_chunk(x) : ForwardDiff.Chunk{CS}()
     tag = __standard_tag(ad.tag, x)
-    return ForwardDiff.JacobianConfig(f, x, ck, tag)
+    return __forwarddiff_jacobian_config(f, x, ck, tag)
 end
-function __get_jacobian_config(ad::AutoForwardDiff{CS}, f!, y, x) where {CS}
+function __get_jacobian_config(ad::AutoForwardDiff{CS}, f!::F, y, x) where {F, CS}
     ck = (CS === nothing || CS ≤ 0) ? __pick_forwarddiff_chunk(x) : ForwardDiff.Chunk{CS}()
     tag = __standard_tag(ad.tag, x)
     return ForwardDiff.JacobianConfig(f!, y, x, ck, tag)
 end
 
+function __forwarddiff_jacobian_config(f::F, x, ck::ForwardDiff.Chunk, tag) where {F}
+    return ForwardDiff.JacobianConfig(f, x, ck, tag)
+end
+function __forwarddiff_jacobian_config(
+        f::F, x::SArray, ck::ForwardDiff.Chunk{N}, tag) where {F, N}
+    seeds = ForwardDiff.construct_seeds(ForwardDiff.Partials{N, eltype(x)})
+    duals = ForwardDiff.Dual{typeof(tag), eltype(x), N}.(x)
+    return ForwardDiff.JacobianConfig{typeof(tag), eltype(x), N, typeof(duals)}(seeds,
+        duals)
+end
+
 function __get_jacobian_config(ad::AutoPolyesterForwardDiff{CS}, args...) where {CS}
     x = last(args)
     return (CS === nothing || CS ≤ 0) ? __pick_forwarddiff_chunk(x) :

From 9203bc3216dd5866a93de10cd57916066071b1f9 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 16 Feb 2024 10:50:33 -0500
Subject: [PATCH 313/375] Julia version not allowed to be changed in patch
 release

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 4065b1fcf..58adfd46a 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.4.3"
+version = "1.5.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From 0166ee965368572b961b802a561258be8c03c84c Mon Sep 17 00:00:00 2001
From: Arno Strouwen <arno.strouwen@telenet.be>
Date: Sun, 18 Feb 2024 13:59:51 +0100
Subject: [PATCH 314/375] [skip ci] update downgrade CI repo

---
 lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
index cbcc6ac06..0d225bb0b 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
@@ -21,9 +21,9 @@ jobs:
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
-      - uses: cjdoris/julia-downgrade-compat-action@v1
+      - uses: julia-actions/julia-downgrade-compat@v1
 #        if: ${{ matrix.version == '1.6' }}
         with:
           skip: Pkg,TOML
       - uses: julia-actions/julia-buildpkg@v1
-      - uses: julia-actions/julia-runtest@v1
\ No newline at end of file
+      - uses: julia-actions/julia-runtest@v1

From e831d1ad65dbf81de23cb76031f9a3d167ba3528 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 26 Feb 2024 13:55:52 -0500
Subject: [PATCH 315/375] Forward Mode overloads for Least Squares Problem

---
 lib/SimpleNonlinearSolve/Project.toml            |  4 +++-
 .../ext/SimpleNonlinearSolveZygoteExt.jl         |  7 +++++++
 lib/SimpleNonlinearSolve/src/ad.jl               |  3 ++-
 .../test/core/least_squares_tests.jl             | 16 ++++++++++++++++
 4 files changed, 28 insertions(+), 2 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 4065b1fcf..390d37a23 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.4.3"
+version = "1.5.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -22,11 +22,13 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [extensions]
 SimpleNonlinearSolveChainRulesCoreExt = "ChainRulesCore"
 SimpleNonlinearSolvePolyesterForwardDiffExt = "PolyesterForwardDiff"
 SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
+SimpleNonlinearSolveZygoteExt = "Zygote"
 
 [compat]
 ADTypes = "0.2.6"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
new file mode 100644
index 000000000..ae3b61352
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
@@ -0,0 +1,7 @@
+module SimpleNonlinearSolveZygoteExt
+
+import SimpleNonlinearSolve
+
+SimpleNonlinearSolve.__is_extension_loaded(::Val{:Zygote}) = true
+
+end
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index b4f7f41b9..579eef120 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -24,7 +24,8 @@ for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
     end
 end
 
-function __nlsolve_ad(prob, alg, args...; kwargs...)
+function __nlsolve_ad(
+        prob::Union{IntervalNonlinearProblem, NonlinearProblem}, alg, args...; kwargs...)
     p = value(prob.p)
     if prob isa IntervalNonlinearProblem
         tspan = value.(prob.tspan)
diff --git a/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl b/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
index 8d99d3544..840a4f277 100644
--- a/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
@@ -12,6 +12,12 @@
         return ŷ .- y_target
     end
 
+    function loss_function!(resid, θ, p)
+        ŷ = true_function(p, θ)
+        @. resid = ŷ - y_target
+        return
+    end
+
     θ_init = θ_true .+ 0.1
     prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
 
@@ -21,4 +27,14 @@
         sol = solve(prob_oop, solver)
         @test norm(sol.resid, Inf) < 1e-12
     end
+
+    prob_iip = NonlinearLeastSquaresProblem(
+        NonlinearFunction{true}(loss_function!, resid_prototype = zeros(length(y_target))), θ_init, x)
+
+    @testset "Solver: $(nameof(typeof(solver)))" for solver in [
+        SimpleNewtonRaphson(AutoForwardDiff()), SimpleGaussNewton(AutoForwardDiff()),
+        SimpleNewtonRaphson(AutoFiniteDiff()), SimpleGaussNewton(AutoFiniteDiff())]
+        sol = solve(prob_iip, solver)
+        @test norm(sol.resid, Inf) < 1e-12
+    end
 end

From 9bf42c35b3ea9be9d6fc2db91a12f06d7ff8025f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 26 Feb 2024 15:20:51 -0500
Subject: [PATCH 316/375] Add implementation for NLLS Forward Mode

---
 lib/SimpleNonlinearSolve/Project.toml         | 18 +++-
 .../ext/SimpleNonlinearSolveZygoteExt.jl      |  7 +-
 .../src/SimpleNonlinearSolve.jl               |  1 +
 lib/SimpleNonlinearSolve/src/ad.jl            | 86 ++++++++++++++++++-
 lib/SimpleNonlinearSolve/src/utils.jl         |  3 +
 .../test/core/aqua_tests.jl                   |  9 ++
 .../test/core/forward_ad_tests.jl             |  4 +-
 .../test/core/rootfind_tests.jl               | 50 +++++------
 8 files changed, 148 insertions(+), 30 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/test/core/aqua_tests.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 390d37a23..a8b406cee 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -8,6 +8,7 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
 ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+DiffResults = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
 FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
@@ -32,26 +33,41 @@ SimpleNonlinearSolveZygoteExt = "Zygote"
 
 [compat]
 ADTypes = "0.2.6"
+AllocCheck = "0.1.1"
 ArrayInterface = "7.7"
+Aqua = "0.8"
+CUDA = "5.2"
 ChainRulesCore = "1.21"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.146"
+DiffResults = "1.1"
 FastClosures = "0.3"
 FiniteDiff = "2.22"
 ForwardDiff = "0.10.36"
 LinearAlgebra = "1.10"
+LinearSolve = "2.25"
 MaybeInplace = "0.1.1"
+NonlinearProblemLibrary = "0.1.2"
+Pkg = "1.10"
+PolyesterForwardDiff = "0.1.1"
 PrecompileTools = "1.2"
+Random = "1.10"
+ReTestItems = "1.23"
 Reexport = "1.2"
 SciMLBase = "2.23"
+SciMLSensitivity = "7.56"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"
+Test = "1.10"
+Zygote = "0.6.69"
 julia = "1.10"
 
 [extras]
 AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
+Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
@@ -67,4 +83,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["AllocCheck", "DiffEqBase", "ForwardDiff", "LinearAlgebra", "LinearSolve", "NonlinearProblemLibrary", "Pkg", "Random", "ReTestItems", "SciMLSensitivity", "StaticArrays", "Zygote", "CUDA", "PolyesterForwardDiff", "Reexport", "Test"]
+test = ["Aqua", "AllocCheck", "DiffEqBase", "ForwardDiff", "LinearAlgebra", "LinearSolve", "NonlinearProblemLibrary", "Pkg", "Random", "ReTestItems", "SciMLSensitivity", "StaticArrays", "Zygote", "CUDA", "PolyesterForwardDiff", "Reexport", "Test", "FiniteDiff"]
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
index ae3b61352..d2caa8998 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
@@ -1,7 +1,12 @@
 module SimpleNonlinearSolveZygoteExt
 
-import SimpleNonlinearSolve
+import SimpleNonlinearSolve, Zygote
 
 SimpleNonlinearSolve.__is_extension_loaded(::Val{:Zygote}) = true
 
+function SimpleNonlinearSolve.__zygote_compute_nlls_vjp(f::F, u, p) where {F}
+    y, pb = Zygote.pullback(Base.Fix2(f, p), u)
+    return 2 .* only(pb(y))
+end
+
 end
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 55b579e95..710f2d6c3 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -11,6 +11,7 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
                        AbstractSafeBestNonlinearTerminationMode,
                        NonlinearSafeTerminationReturnCode, get_termination_mode,
                        NONLINEARSOLVE_DEFAULT_NORM
+    import DiffResults
     import ForwardDiff: Dual
     import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
     import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace, _unwrap_val
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 579eef120..c8683ac71 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -5,8 +5,17 @@ function SciMLBase.solve(
     sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
     dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
     return SciMLBase.build_solution(
-        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats,
-        sol.original)
+        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
+end
+
+function SciMLBase.solve(
+        prob::NonlinearLeastSquaresProblem{<:AbstractArray,
+            iip, <:Union{<:AbstractArray{<:Dual{T, V, P}}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) where {T, V, P, iip}
+    sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
+    dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
+    return SciMLBase.build_solution(
+        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
 end
 
 for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
@@ -56,6 +65,79 @@ function __nlsolve_ad(
     return sol, partials
 end
 
+function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs...)
+    p = value(prob.p)
+    u0 = value(prob.u0)
+    newprob = NonlinearLeastSquaresProblem(prob.f, u0, p; prob.kwargs...)
+
+    sol = solve(newprob, alg, args...; kwargs...)
+
+    uu = sol.u
+
+    if !SciMLBase.has_jac(prob.f)
+        if isinplace(prob)
+            _F = @closure (du, u, p) -> begin
+                resid = similar(du, length(sol.resid))
+                res = DiffResults.DiffResult(
+                    resid, similar(du, length(sol.resid), length(u)))
+                _f = @closure (du, u) -> prob.f(du, u, p)
+                ForwardDiff.jacobian!(res, _f, resid, u)
+                mul!(reshape(du, 1, :), vec(DiffResults.value(res))',
+                    DiffResults.jacobian(res), 2, false)
+                return nothing
+            end
+        else
+            # For small problems, nesting ForwardDiff is actually quite fast
+            if __is_extension_loaded(Val(:Zygote)) && (length(uu) + length(sol.resid) ≥ 50)
+                _F = @closure (u, p) -> __zygote_compute_nlls_vjp(prob.f, u, p)
+            else
+                _F = @closure (u, p) -> begin
+                    T = promote_type(eltype(u), eltype(p))
+                    res = DiffResults.DiffResult(
+                        similar(u, T, size(sol.resid)), similar(
+                            u, T, length(sol.resid), length(u)))
+                    ForwardDiff.jacobian!(res, Base.Fix2(prob.f, p), u)
+                    return reshape(
+                        2 .* vec(DiffResults.value(res))' * DiffResults.jacobian(res),
+                        size(u))
+                end
+            end
+        end
+    else
+        if isinplace(prob)
+            _F = @closure (du, u, p) -> begin
+                J = similar(du, length(sol.resid), length(u))
+                prob.jac(J, u, p)
+                resid = similar(du, length(sol.resid))
+                prob.f(resid, u, p)
+                mul!(reshape(du, 1, :), vec(resid)', J, 2, false)
+                return nothing
+            end
+        else
+            _F = @closure (u, p) -> begin
+                return reshape(2 .* vec(prob.f(u, p))' * prob.jac(u, p), size(u))
+            end
+        end
+    end
+
+    f_p = __nlsolve_∂f_∂p(prob, _F, uu, p)
+    f_x = __nlsolve_∂f_∂u(prob, _F, uu, p)
+
+    z_arr = -f_x \ f_p
+
+    pp = prob.p
+    sumfun = ((z, p),) -> map(zᵢ -> zᵢ * ForwardDiff.partials(p), z)
+    if uu isa Number
+        partials = sum(sumfun, zip(z_arr, pp))
+    elseif p isa Number
+        partials = sumfun((z_arr, pp))
+    else
+        partials = sum(sumfun, zip(eachcol(z_arr), pp))
+    end
+
+    return sol, partials
+end
+
 @inline function __nlsolve_∂f_∂p(prob, f::F, u, p) where {F}
     if isinplace(prob)
         __f = p -> begin
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index e89794b4c..7a52fdedc 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -388,3 +388,6 @@ function __get_tolerance(x::Union{SArray, Number}, ::Nothing, ::Type{T}) where {
     η = real(oneunit(T)) * (eps(real(one(T))))^(real(T)(0.8))
     return T(η)
 end
+
+# Extension
+function __zygote_compute_nlls_vjp end
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl b/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl
new file mode 100644
index 000000000..72437ff37
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl
@@ -0,0 +1,9 @@
+@testitem "Aqua" begin
+    using Aqua
+
+    Aqua.test_all(SimpleNonlinearSolve; piracies = false, ambiguities = false)
+    Aqua.test_piracies(SimpleNonlinearSolve;
+        treat_as_own = [
+            NonlinearProblem, NonlinearLeastSquaresProblem, IntervalNonlinearProblem])
+    Aqua.test_ambiguities(SimpleNonlinearSolve; recursive = false)
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
index e7fc44f02..3d2c87da4 100644
--- a/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
@@ -1,4 +1,4 @@
-@testsetup module ForwardADTesting
+@testsetup module ForwardADRootfindingTesting
 using Reexport
 @reexport using ForwardDiff, SimpleNonlinearSolve, StaticArrays, LinearAlgebra
 import SimpleNonlinearSolve: AbstractSimpleNonlinearSolveAlgorithm
@@ -40,7 +40,7 @@ __compatible(::SimpleHalley, ::Val{:iip}) = false
 export test_f, test_f!, jacobian_f, solve_with, __compatible
 end
 
-@testitem "ForwardDiff.jl Integration" setup=[ForwardADTesting] begin
+@testitem "ForwardDiff.jl Integration: Rootfinding" setup=[ForwardADRootfindingTesting] begin
     @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
         SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
         SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleDFSane())
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index f3ac188a9..726a6dda7 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -134,31 +134,33 @@ end
 end
 
 @testitem "Allocation Checks" setup=[RootfindingTesting] begin
-    @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
-        SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
-        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
-        SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)),
-        SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
-        @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
-
-        nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
-        nlprob_sa = NonlinearProblem{false}(quadratic_f, @SVector[1.0, 1.0], 2.0)
-
-        try
-            nlsolve(nlprob_scalar, alg)
-            @test true
-        catch e
-            @error e
-            @test false
-        end
+    if Sys.islinux()  # Very slow on other OS
+        @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
+            SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
+            SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+            SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)),
+            SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
+            @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
+
+            nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
+            nlprob_sa = NonlinearProblem{false}(quadratic_f, @SVector[1.0, 1.0], 2.0)
+
+            try
+                nlsolve(nlprob_scalar, alg)
+                @test true
+            catch e
+                @error e
+                @test false
+            end
 
-        # ForwardDiff allocates for hessian since we don't propagate the chunksize
-        try
-            nlsolve(nlprob_sa, alg)
-            @test true
-        catch e
-            @error e
-            @test false broken=(alg isa SimpleHalley)
+            # ForwardDiff allocates for hessian since we don't propagate the chunksize
+            try
+                nlsolve(nlprob_sa, alg)
+                @test true
+            catch e
+                @error e
+                @test false broken=(alg isa SimpleHalley)
+            end
         end
     end
 end

From c7930cc0c63d7d267d88f8bb00d273559874d58e Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 26 Feb 2024 16:02:55 -0500
Subject: [PATCH 317/375] Use custom vjp

---
 lib/SimpleNonlinearSolve/src/ad.jl            |  49 +++++---
 .../test/core/forward_ad_tests.jl             | 118 ++++++++++++++++++
 2 files changed, 151 insertions(+), 16 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index c8683ac71..d4e091c43 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -74,7 +74,39 @@ function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs..
 
     uu = sol.u
 
-    if !SciMLBase.has_jac(prob.f)
+    # First check for custom `vjp` then custom `Jacobian` and if nothing is provided use
+    # nested autodiff as the last resort
+    if SciMLBase.has_vjp(prob.f)
+        if isinplace(prob)
+            _F = @closure (du, u, p) -> begin
+                resid = similar(du, length(sol.resid))
+                prob.f(resid, u, p)
+                prob.f.vjp(du, resid, u, p)
+                du .*= 2
+                return nothing
+            end
+        else
+            _F = @closure (u, p) -> begin
+                resid = prob.f(u, p)
+                return reshape(2 .* prob.f.vjp(resid, u, p), size(u))
+            end
+        end
+    elseif SciMLBase.has_jac(prob.f)
+        if isinplace(prob)
+            _F = @closure (du, u, p) -> begin
+                J = similar(du, length(sol.resid), length(u))
+                prob.f.jac(J, u, p)
+                resid = similar(du, length(sol.resid))
+                prob.f(resid, u, p)
+                mul!(reshape(du, 1, :), vec(resid)', J, 2, false)
+                return nothing
+            end
+        else
+            _F = @closure (u, p) -> begin
+                return reshape(2 .* vec(prob.f(u, p))' * prob.f.jac(u, p), size(u))
+            end
+        end
+    else
         if isinplace(prob)
             _F = @closure (du, u, p) -> begin
                 resid = similar(du, length(sol.resid))
@@ -103,21 +135,6 @@ function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs..
                 end
             end
         end
-    else
-        if isinplace(prob)
-            _F = @closure (du, u, p) -> begin
-                J = similar(du, length(sol.resid), length(u))
-                prob.jac(J, u, p)
-                resid = similar(du, length(sol.resid))
-                prob.f(resid, u, p)
-                mul!(reshape(du, 1, :), vec(resid)', J, 2, false)
-                return nothing
-            end
-        else
-            _F = @closure (u, p) -> begin
-                return reshape(2 .* vec(prob.f(u, p))' * prob.jac(u, p), size(u))
-            end
-        end
     end
 
     f_p = __nlsolve_∂f_∂p(prob, _F, uu, p)
diff --git a/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
index 3d2c87da4..e0852ba53 100644
--- a/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
@@ -88,3 +88,121 @@ end
         end
     end
 end
+
+@testsetup module ForwardADNLLSTesting
+using Reexport
+@reexport using ForwardDiff, FiniteDiff, SimpleNonlinearSolve, StaticArrays, LinearAlgebra,
+                Zygote
+
+true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
+
+const θ_true = [1.0, 0.1, 2.0, 0.5]
+const x = [-1.0, -0.5, 0.0, 0.5, 1.0]
+const y_target = true_function(x, θ_true)
+
+function loss_function(θ, p)
+    ŷ = true_function(p, θ)
+    return ŷ .- y_target
+end
+
+function loss_function_jac(θ, p)
+    return ForwardDiff.jacobian(θ -> loss_function(θ, p), θ)
+end
+
+loss_function_vjp(v, θ, p) = reshape(vec(v)' * loss_function_jac(θ, p), size(θ))
+
+function loss_function!(resid, θ, p)
+    ŷ = true_function(p, θ)
+    @. resid = ŷ - y_target
+    return
+end
+
+function loss_function_jac!(J, θ, p)
+    J .= ForwardDiff.jacobian(θ -> loss_function(θ, p), θ)
+    return
+end
+
+function loss_function_vjp!(vJ, v, θ, p)
+    vec(vJ) .= reshape(vec(v)' * loss_function_jac(θ, p), size(θ))
+    return
+end
+
+θ_init = θ_true .+ 0.1
+
+export loss_function, loss_function!, loss_function_jac, loss_function_vjp,
+       loss_function_jac!, loss_function_vjp!, θ_init, x, y_target
+end
+
+@testitem "ForwardDiff.jl Integration: NLLS" setup=[ForwardADNLLSTesting] begin
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleNewtonRaphson(), SimpleGaussNewton(),
+        SimpleNewtonRaphson(AutoFiniteDiff()), SimpleGaussNewton(AutoFiniteDiff()))
+        function obj_1(p)
+            prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, p)
+            sol = solve(prob_oop, alg)
+            return sum(abs2, sol.u)
+        end
+
+        function obj_2(p)
+            ff = NonlinearFunction{false}(loss_function; jac = loss_function_jac)
+            prob_oop = NonlinearLeastSquaresProblem{false}(ff, θ_init, p)
+            sol = solve(prob_oop, alg)
+            return sum(abs2, sol.u)
+        end
+
+        function obj_3(p)
+            ff = NonlinearFunction{false}(loss_function; vjp = loss_function_vjp)
+            prob_oop = NonlinearLeastSquaresProblem{false}(ff, θ_init, p)
+            sol = solve(prob_oop, alg)
+            return sum(abs2, sol.u)
+        end
+
+        finitediff = FiniteDiff.finite_difference_gradient(obj_1, x)
+
+        fdiff1 = ForwardDiff.gradient(obj_1, x)
+        fdiff2 = ForwardDiff.gradient(obj_2, x)
+        fdiff3 = ForwardDiff.gradient(obj_3, x)
+
+        @test finitediff≈fdiff1 atol=1e-5
+        @test finitediff≈fdiff2 atol=1e-5
+        @test finitediff≈fdiff3 atol=1e-5
+        @test fdiff1 ≈ fdiff2 ≈ fdiff3
+
+        function obj_4(p)
+            prob_iip = NonlinearLeastSquaresProblem(
+                NonlinearFunction{true}(
+                    loss_function!; resid_prototype = zeros(length(y_target))), θ_init, p)
+            sol = solve(prob_iip, alg)
+            return sum(abs2, sol.u)
+        end
+
+        function obj_5(p)
+            ff = NonlinearFunction{true}(
+                loss_function!; resid_prototype = zeros(length(y_target)), jac = loss_function_jac!)
+            prob_iip = NonlinearLeastSquaresProblem(
+                ff, θ_init, p)
+            sol = solve(prob_iip, alg)
+            return sum(abs2, sol.u)
+        end
+
+        function obj_6(p)
+            ff = NonlinearFunction{true}(
+                loss_function!; resid_prototype = zeros(length(y_target)), vjp = loss_function_vjp!)
+            prob_iip = NonlinearLeastSquaresProblem(
+                ff, θ_init, p)
+            sol = solve(prob_iip, alg)
+            return sum(abs2, sol.u)
+        end
+
+        finitediff = FiniteDiff.finite_difference_gradient(obj_4, x)
+
+        fdiff4 = ForwardDiff.gradient(obj_4, x)
+        fdiff5 = ForwardDiff.gradient(obj_5, x)
+        fdiff6 = ForwardDiff.gradient(obj_6, x)
+
+        @test finitediff≈fdiff4 atol=1e-5
+        @test finitediff≈fdiff5 atol=1e-5
+        @test finitediff≈fdiff6 atol=1e-5
+        @test fdiff4 ≈ fdiff5 ≈ fdiff6
+    end
+end

From 2b8b5e42d5281364fe04efd6552e3381160329a5 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 26 Feb 2024 16:06:57 -0500
Subject: [PATCH 318/375] Formatting and Downgrade

---
 lib/SimpleNonlinearSolve/Project.toml                         | 4 ++--
 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl | 2 +-
 lib/SimpleNonlinearSolve/src/utils.jl                         | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index a8b406cee..a73771cce 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -37,7 +37,7 @@ AllocCheck = "0.1.1"
 ArrayInterface = "7.7"
 Aqua = "0.8"
 CUDA = "5.2"
-ChainRulesCore = "1.21"
+ChainRulesCore = "1.22"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.146"
 DiffResults = "1.1"
@@ -54,7 +54,7 @@ PrecompileTools = "1.2"
 Random = "1.10"
 ReTestItems = "1.23"
 Reexport = "1.2"
-SciMLBase = "2.23"
+SciMLBase = "2.26.3"
 SciMLSensitivity = "7.56"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
index d2caa8998..b29a1529a 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
@@ -9,4 +9,4 @@ function SimpleNonlinearSolve.__zygote_compute_nlls_vjp(f::F, u, p) where {F}
     return 2 .* only(pb(y))
 end
 
-end
\ No newline at end of file
+end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 7a52fdedc..2876cedb2 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -390,4 +390,4 @@ function __get_tolerance(x::Union{SArray, Number}, ::Nothing, ::Type{T}) where {
 end
 
 # Extension
-function __zygote_compute_nlls_vjp end
\ No newline at end of file
+function __zygote_compute_nlls_vjp end

From 96874e9ad9103436e554221fbcbe337609fd6c9e Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 10 Mar 2024 18:40:03 -0400
Subject: [PATCH 319/375] Dont default to Polyester

---
 lib/SimpleNonlinearSolve/src/nlsolve/halley.jl |  2 +-
 lib/SimpleNonlinearSolve/src/utils.jl          | 15 +++------------
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 44877e097..462332280 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -33,7 +33,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
     fx = _get_fx(prob, x)
     T = eltype(x)
 
-    autodiff = __get_concrete_autodiff(prob, alg.autodiff; polyester = Val(false))
+    autodiff = __get_concrete_autodiff(prob, alg.autodiff)
     abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
         termination_condition)
 
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 2876cedb2..38be3437c 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -365,18 +365,9 @@ end
 
 # Decide which AD backend to use
 @inline __get_concrete_autodiff(prob, ad::ADTypes.AbstractADType; kwargs...) = ad
-@inline function __get_concrete_autodiff(prob, ::Nothing; polyester::Val{P} = Val(true),
-        kwargs...) where {P}
-    if ForwardDiff.can_dual(eltype(prob.u0))
-        if P && __is_extension_loaded(Val(:PolyesterForwardDiff)) &&
-           !(prob.u0 isa Number) && ArrayInterface.can_setindex(prob.u0)
-            return AutoPolyesterForwardDiff()
-        else
-            return AutoForwardDiff()
-        end
-    else
-        return AutoFiniteDiff()
-    end
+@inline function __get_concrete_autodiff(prob, ::Nothing; kwargs...)
+    return ifelse(
+        ForwardDiff.can_dual(eltype(prob.u0)), AutoForwardDiff(), AutoFiniteDiff())
 end
 
 @inline __reshape(x::Number, args...) = x

From 73038b75ed259e14c6e9da1b2b94c97b2943525f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 10 Mar 2024 20:06:11 -0400
Subject: [PATCH 320/375] Add ReverseDiff and Tracker entry points

---
 lib/SimpleNonlinearSolve/Project.toml         | 14 ++++-
 .../SimpleNonlinearSolveChainRulesCoreExt.jl  |  7 +--
 .../ext/SimpleNonlinearSolveReverseDiffExt.jl | 60 +++++++++++++++++++
 .../ext/SimpleNonlinearSolveTrackerExt.jl     | 42 +++++++++++++
 .../src/SimpleNonlinearSolve.jl               |  4 +-
 .../test/core/adjoint_tests.jl                | 13 +++-
 6 files changed, 128 insertions(+), 12 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index a73771cce..225b36090 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.5.0"
+version = "1.6.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -22,20 +22,24 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 [weakdeps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
+ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [extensions]
 SimpleNonlinearSolveChainRulesCoreExt = "ChainRulesCore"
 SimpleNonlinearSolvePolyesterForwardDiffExt = "PolyesterForwardDiff"
+SimpleNonlinearSolveReverseDiffExt = "ReverseDiff"
 SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
+SimpleNonlinearSolveTrackerExt = "Tracker"
 SimpleNonlinearSolveZygoteExt = "Zygote"
 
 [compat]
 ADTypes = "0.2.6"
 AllocCheck = "0.1.1"
-ArrayInterface = "7.7"
 Aqua = "0.8"
+ArrayInterface = "7.7"
 CUDA = "5.2"
 ChainRulesCore = "1.22"
 ConcreteStructs = "0.2.3"
@@ -54,11 +58,13 @@ PrecompileTools = "1.2"
 Random = "1.10"
 ReTestItems = "1.23"
 Reexport = "1.2"
+ReverseDiff = "1.15"
 SciMLBase = "2.26.3"
 SciMLSensitivity = "7.56"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"
 Test = "1.10"
+Tracker = "0.2.32"
 Zygote = "0.6.69"
 julia = "1.10"
 
@@ -77,10 +83,12 @@ PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
+ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
 SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["Aqua", "AllocCheck", "DiffEqBase", "ForwardDiff", "LinearAlgebra", "LinearSolve", "NonlinearProblemLibrary", "Pkg", "Random", "ReTestItems", "SciMLSensitivity", "StaticArrays", "Zygote", "CUDA", "PolyesterForwardDiff", "Reexport", "Test", "FiniteDiff"]
+test = ["Aqua", "AllocCheck", "DiffEqBase", "ForwardDiff", "LinearAlgebra", "LinearSolve", "NonlinearProblemLibrary", "Pkg", "Random", "ReTestItems", "SciMLSensitivity", "StaticArrays", "Zygote", "CUDA", "PolyesterForwardDiff", "Reexport", "Test", "FiniteDiff", "ReverseDiff", "Tracker"]
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
index 23cd25f33..dc84cb3e8 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
@@ -5,14 +5,13 @@ using ChainRulesCore, DiffEqBase, SciMLBase, SimpleNonlinearSolve
 # The expectation here is that no-one is using this directly inside a GPU kernel. We can
 # eventually lift this requirement using a custom adjoint
 function ChainRulesCore.rrule(::typeof(SimpleNonlinearSolve.__internal_solve_up),
-        prob::NonlinearProblem,
-        sensealg::Union{Nothing, DiffEqBase.AbstractSensitivityAlgorithm}, u0, u0_changed,
-        p, p_changed, alg, args...; kwargs...)
+        prob::NonlinearProblem, sensealg, u0, u0_changed, p, p_changed, alg, args...;
+        kwargs...)
     out, ∇internal = DiffEqBase._solve_adjoint(prob, sensealg, u0, p,
         SciMLBase.ChainRulesOriginator(), alg, args...; kwargs...)
     function ∇__internal_solve_up(Δ)
         ∂f, ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
-        return (∂f, ∂prob, ∂sensealg, ∂u0, NoTangent(), ∂p, NoTangent(), ∂originator,
+        return (∂f, ∂prob, ∂sensealg, ∂u0, NoTangent(), ∂p, NoTangent(), NoTangent(),
             ∂args...)
     end
     return out, ∇__internal_solve_up
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
new file mode 100644
index 000000000..e0bbda27e
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
@@ -0,0 +1,60 @@
+module SimpleNonlinearSolveReverseDiffExt
+
+using ArrayInterface, DiffEqBase, ReverseDiff, SciMLBase, SimpleNonlinearSolve
+import ReverseDiff: TrackedArray, TrackedReal
+import SimpleNonlinearSolve: __internal_solve_up
+
+function __internal_solve_up(
+        prob::NonlinearProblem, sensealg, u0::TrackedArray, u0_changed,
+        p::TrackedArray, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
+end
+
+function __internal_solve_up(
+        prob::NonlinearProblem, sensealg, u0, u0_changed,
+        p::TrackedArray, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
+end
+
+function __internal_solve_up(
+        prob::NonlinearProblem, sensealg, u0::TrackedArray, u0_changed,
+        p, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
+end
+
+function __internal_solve_up(prob::NonlinearProblem, sensealg,
+        u0::AbstractArray{<:TrackedReal}, u0_changed, p::AbstractArray{<:TrackedReal},
+        p_changed, alg, args...; kwargs...)
+    return __internal_solve_up(
+        prob, sensealg, ArrayInterface.aos_to_soa(u0), true,
+        ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
+end
+
+function __internal_solve_up(prob::NonlinearProblem, sensealg, u0, u0_changed,
+        p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
+    return __internal_solve_up(
+        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
+end
+
+function __internal_solve_up(prob::NonlinearProblem, sensealg,
+        u0::AbstractArray{<:TrackedReal}, u0_changed, p, p_changed, alg, args...; kwargs...)
+    return __internal_solve_up(
+        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
+end
+
+ReverseDiff.@grad function __internal_solve_up(
+        prob::NonlinearProblem, sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+    out, ∇internal = DiffEqBase._solve_adjoint(
+        prob, sensealg, ReverseDiff.value(u0), ReverseDiff.value(p),
+        SciMLBase.ReverseDiffOriginator(), alg, args...; kwargs...)
+    function ∇__internal_solve_up(_args...)
+        ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(_args...)
+        return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
+    end
+    return Array(out), ∇__internal_solve_up
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
new file mode 100644
index 000000000..61ce14645
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
@@ -0,0 +1,42 @@
+module SimpleNonlinearSolveTrackerExt
+
+using DiffEqBase, SciMLBase, SimpleNonlinearSolve, Tracker
+
+function SimpleNonlinearSolve.__internal_solve_up(prob::NonlinearProblem,
+        sensealg, u0::TrackedArray, u0_changed, p, p_changed, alg, args...; kwargs...)
+    return Tracker.track(
+        SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0, u0_changed,
+        p, p_changed, alg, args...; kwargs...)
+end
+
+function SimpleNonlinearSolve.__internal_solve_up(
+        prob::NonlinearProblem, sensealg, u0::TrackedArray, u0_changed,
+        p::TrackedArray, p_changed, alg, args...; kwargs...)
+    return Tracker.track(
+        SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0, u0_changed,
+        p, p_changed, alg, args...; kwargs...)
+end
+
+function SimpleNonlinearSolve.__internal_solve_up(prob::NonlinearProblem,
+        sensealg, u0, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
+    return Tracker.track(
+        SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0, u0_changed,
+        p, p_changed, alg, args...; kwargs...)
+end
+
+Tracker.@grad function SimpleNonlinearSolve.__internal_solve_up(_prob::NonlinearProblem,
+        sensealg, u0_, u0_changed, p_, p_changed, alg, args...; kwargs...)
+    u0, p = Tracker.data(u0_), Tracker.data(p_)
+    prob = remake(_prob; u0, p)
+    out, ∇internal = DiffEqBase._solve_adjoint(prob, sensealg, u0, p,
+        SciMLBase.TrackerOriginator(), alg, args...; kwargs...)
+
+    function ∇__internal_solve_up(Δ)
+        ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
+        return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
+    end
+
+    return out, ∇__internal_solve_up
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 710f2d6c3..3a4e5ebdb 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -71,8 +71,8 @@ function SciMLBase.solve(
         alg, args...; prob.kwargs..., kwargs...)
 end
 
-function __internal_solve_up(_prob::NonlinearProblem, sensealg, u0, u0_changed, p,
-        p_changed, alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...)
+function __internal_solve_up(_prob::NonlinearProblem, sensealg, u0, u0_changed,
+        p, p_changed, alg, args...; kwargs...)
     prob = u0_changed || p_changed ? remake(_prob; u0, p) : _prob
     return SciMLBase.__solve(prob, alg, args...; kwargs...)
 end
diff --git a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
index f96491449..74d745f65 100644
--- a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
@@ -1,14 +1,21 @@
 @testitem "Simple Adjoint Test" begin
-    using ForwardDiff, SciMLSensitivity, Zygote
+    using ForwardDiff, ReverseDiff, SciMLSensitivity, Tracker, Zygote
 
     ff(u, p) = u .^ 2 .- p
 
     function solve_nlprob(p)
         prob = NonlinearProblem{false}(ff, [1.0, 2.0], p)
-        return sum(abs2, solve(prob, SimpleNewtonRaphson()).u)
+        sol = solve(prob, SimpleNewtonRaphson())
+        res = sol isa AbstractArray ? sol : sol.u
+        return sum(abs2, res)
     end
 
     p = [3.0, 2.0]
 
-    @test only(Zygote.gradient(solve_nlprob, p)) ≈ ForwardDiff.gradient(solve_nlprob, p)
+    ∂p_zygote = only(Zygote.gradient(solve_nlprob, p))
+    ∂p_forwarddiff = ForwardDiff.gradient(solve_nlprob, p)
+    ∂p_tracker = Tracker.data(only(Tracker.gradient(solve_nlprob, p)))
+    ∂p_reversediff = ReverseDiff.gradient(solve_nlprob, p)
+
+    @test ∂p_zygote ≈ ∂p_forwarddiff ≈ ∂p_tracker ≈ ∂p_reversediff
 end

From ec757e3f6a4603939a7a3b741865bd6131c01737 Mon Sep 17 00:00:00 2001
From: Hendrik Ranocha <ranocha@users.noreply.github.com>
Date: Wed, 13 Mar 2024 10:22:46 +0100
Subject: [PATCH 321/375] fix docstring of ITP

---
 lib/SimpleNonlinearSolve/src/bracketing/itp.jl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index 3f2069b72..2926dfd7a 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -14,13 +14,13 @@ I. F. D. Oliveira and R. H. C. Takahashi.
 
 The following keyword parameters are accepted.
 
-  - `n₀::Int = 1`, the 'slack'. Must not be negative. When n₀ = 0 the worst-case is
+  - `n₀::Int = 10`, the 'slack'. Must not be negative. When n₀ = 0 the worst-case is
     identical to that of bisection, but increacing n₀ provides greater oppotunity for
     superlinearity.
-  - `κ₁::Float64 = 0.1`. Must not be negative. The recomended value is `0.2/(x₂ - x₁)`.
+  - `κ₁::Float64 = 0.007`. Must not be negative. The recomended value is `0.2/(x₂ - x₁)`.
     Lower values produce tighter asymptotic behaviour, while higher values improve the
     steady-state behaviour when truncation is not helpful.
-  - `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62). Higher values allow for a greater
+  - `κ₂::Real = 1.5`. Must lie in [1, 1+ϕ ≈ 2.62). Higher values allow for a greater
     convergence rate, but also make the method more succeptable to worst-case performance.
     In practice, κ=1,2 seems to work well due to the computational simplicity, as κ₂ is used
     as an exponent in the method.

From 36814a2093ecfb9b715413749b1c249fa56227e7 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 31 Mar 2024 12:39:35 -0400
Subject: [PATCH 322/375] Use the different norms for termination

---
 lib/SimpleNonlinearSolve/Manifest.toml        | 781 ++++++++++++++++++
 lib/SimpleNonlinearSolve/Project.toml         |   4 +-
 .../src/nlsolve/broyden.jl                    |   2 +-
 .../src/nlsolve/dfsane.jl                     |   2 +-
 .../src/nlsolve/halley.jl                     |   2 +-
 .../src/nlsolve/klement.jl                    |   2 +-
 .../src/nlsolve/lbroyden.jl                   |   2 +-
 .../src/nlsolve/raphson.jl                    |   2 +-
 .../src/nlsolve/trustRegion.jl                |   2 +-
 lib/SimpleNonlinearSolve/src/utils.jl         |  54 +-
 10 files changed, 815 insertions(+), 38 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/Manifest.toml

diff --git a/lib/SimpleNonlinearSolve/Manifest.toml b/lib/SimpleNonlinearSolve/Manifest.toml
new file mode 100644
index 000000000..a3a6bb313
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/Manifest.toml
@@ -0,0 +1,781 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.10.2"
+manifest_format = "2.0"
+project_hash = "fb617ddc9060eafbe5e2a53eddeb505916c65ad4"
+
+[[deps.ADTypes]]
+git-tree-sha1 = "016833eb52ba2d6bea9fcb50ca295980e728ee24"
+uuid = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+version = "0.2.7"
+
+[[deps.Accessors]]
+deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "LinearAlgebra", "MacroTools", "Markdown", "Test"]
+git-tree-sha1 = "c0d491ef0b135fd7d63cbc6404286bc633329425"
+uuid = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
+version = "0.1.36"
+
+    [deps.Accessors.extensions]
+    AccessorsAxisKeysExt = "AxisKeys"
+    AccessorsIntervalSetsExt = "IntervalSets"
+    AccessorsStaticArraysExt = "StaticArrays"
+    AccessorsStructArraysExt = "StructArrays"
+    AccessorsUnitfulExt = "Unitful"
+
+    [deps.Accessors.weakdeps]
+    AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5"
+    IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
+    Requires = "ae029012-a4dd-5104-9daa-d747884805df"
+    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+    StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
+    Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
+
+[[deps.Adapt]]
+deps = ["LinearAlgebra", "Requires"]
+git-tree-sha1 = "6a55b747d1812e699320963ffde36f1ebdda4099"
+uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
+version = "4.0.4"
+
+    [deps.Adapt.extensions]
+    AdaptStaticArraysExt = "StaticArrays"
+
+    [deps.Adapt.weakdeps]
+    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+
+[[deps.ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+version = "1.1.1"
+
+[[deps.ArrayInterface]]
+deps = ["Adapt", "LinearAlgebra", "SparseArrays", "SuiteSparse"]
+git-tree-sha1 = "44691067188f6bd1b2289552a23e4b7572f4528d"
+uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
+version = "7.9.0"
+
+    [deps.ArrayInterface.extensions]
+    ArrayInterfaceBandedMatricesExt = "BandedMatrices"
+    ArrayInterfaceBlockBandedMatricesExt = "BlockBandedMatrices"
+    ArrayInterfaceCUDAExt = "CUDA"
+    ArrayInterfaceChainRulesExt = "ChainRules"
+    ArrayInterfaceGPUArraysCoreExt = "GPUArraysCore"
+    ArrayInterfaceReverseDiffExt = "ReverseDiff"
+    ArrayInterfaceStaticArraysCoreExt = "StaticArraysCore"
+    ArrayInterfaceTrackerExt = "Tracker"
+
+    [deps.ArrayInterface.weakdeps]
+    BandedMatrices = "aae01518-5342-5314-be14-df237901396f"
+    BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0"
+    CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+    ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2"
+    GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527"
+    ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
+    StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
+    Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
+
+[[deps.Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
+[[deps.Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[deps.BitTwiddlingConvenienceFunctions]]
+deps = ["Static"]
+git-tree-sha1 = "0c5f81f47bbbcf4aea7b2959135713459170798b"
+uuid = "62783981-4cbd-42fc-bca8-16325de8dc4b"
+version = "0.1.5"
+
+[[deps.CPUSummary]]
+deps = ["CpuId", "IfElse", "PrecompileTools", "Static"]
+git-tree-sha1 = "601f7e7b3d36f18790e2caf83a882d88e9b71ff1"
+uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9"
+version = "0.2.4"
+
+[[deps.CloseOpenIntervals]]
+deps = ["Static", "StaticArrayInterface"]
+git-tree-sha1 = "70232f82ffaab9dc52585e0dd043b5e0c6b714f1"
+uuid = "fb6a15b2-703c-40df-9091-08a04967cfa9"
+version = "0.1.12"
+
+[[deps.CommonSolve]]
+git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c"
+uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
+version = "0.2.4"
+
+[[deps.CommonSubexpressions]]
+deps = ["MacroTools", "Test"]
+git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7"
+uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
+version = "0.3.0"
+
+[[deps.Compat]]
+deps = ["TOML", "UUIDs"]
+git-tree-sha1 = "c955881e3c981181362ae4088b35995446298b80"
+uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
+version = "4.14.0"
+weakdeps = ["Dates", "LinearAlgebra"]
+
+    [deps.Compat.extensions]
+    CompatLinearAlgebraExt = "LinearAlgebra"
+
+[[deps.CompilerSupportLibraries_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
+version = "1.1.0+0"
+
+[[deps.CompositionsBase]]
+git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad"
+uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b"
+version = "0.1.2"
+weakdeps = ["InverseFunctions"]
+
+    [deps.CompositionsBase.extensions]
+    CompositionsBaseInverseFunctionsExt = "InverseFunctions"
+
+[[deps.ConcreteStructs]]
+git-tree-sha1 = "f749037478283d372048690eb3b5f92a79432b34"
+uuid = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+version = "0.2.3"
+
+[[deps.ConstructionBase]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "260fd2400ed2dab602a7c15cf10c1933c59930a2"
+uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
+version = "1.5.5"
+
+    [deps.ConstructionBase.extensions]
+    ConstructionBaseIntervalSetsExt = "IntervalSets"
+    ConstructionBaseStaticArraysExt = "StaticArrays"
+
+    [deps.ConstructionBase.weakdeps]
+    IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
+    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+
+[[deps.CpuId]]
+deps = ["Markdown"]
+git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406"
+uuid = "adafc99b-e345-5852-983c-f28acb93d879"
+version = "0.3.1"
+
+[[deps.DataAPI]]
+git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe"
+uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+version = "1.16.0"
+
+[[deps.DataStructures]]
+deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
+git-tree-sha1 = "0f4b5d62a88d8f59003e43c25a8a90de9eb76317"
+uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+version = "0.18.18"
+
+[[deps.DataValueInterfaces]]
+git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
+uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
+version = "1.0.0"
+
+[[deps.Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[deps.DiffEqBase]]
+deps = ["ArrayInterface", "ConcreteStructs", "DataStructures", "DocStringExtensions", "EnumX", "EnzymeCore", "FastBroadcast", "FastClosures", "ForwardDiff", "FunctionWrappers", "FunctionWrappersWrappers", "LinearAlgebra", "Logging", "Markdown", "MuladdMacro", "Parameters", "PreallocationTools", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Static", "StaticArraysCore", "Statistics", "Tricks", "TruncatedStacktraces"]
+git-tree-sha1 = "4fa023dbb15b3485426bbc6c43e030c14250d664"
+repo-rev = "ap/nlls"
+repo-url = "https://github.com/SciML/DiffEqBase.jl.git"
+uuid = "2b5f629d-d688-5b77-993f-72d75c75574e"
+version = "6.149.0"
+
+    [deps.DiffEqBase.extensions]
+    DiffEqBaseChainRulesCoreExt = "ChainRulesCore"
+    DiffEqBaseDistributionsExt = "Distributions"
+    DiffEqBaseEnzymeExt = ["ChainRulesCore", "Enzyme"]
+    DiffEqBaseGeneralizedGeneratedExt = "GeneralizedGenerated"
+    DiffEqBaseMPIExt = "MPI"
+    DiffEqBaseMeasurementsExt = "Measurements"
+    DiffEqBaseMonteCarloMeasurementsExt = "MonteCarloMeasurements"
+    DiffEqBaseReverseDiffExt = "ReverseDiff"
+    DiffEqBaseTrackerExt = "Tracker"
+    DiffEqBaseUnitfulExt = "Unitful"
+
+    [deps.DiffEqBase.weakdeps]
+    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+    Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+    Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
+    GeneralizedGenerated = "6b9d7cbe-bcb9-11e9-073f-15a7a543e2eb"
+    MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
+    Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7"
+    MonteCarloMeasurements = "0987c9cc-fe09-11e8-30f0-b96dd679fdca"
+    ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
+    Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
+    Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
+
+[[deps.DiffResults]]
+deps = ["StaticArraysCore"]
+git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621"
+uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
+version = "1.1.0"
+
+[[deps.DiffRules]]
+deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"]
+git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272"
+uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
+version = "1.15.1"
+
+[[deps.Distributed]]
+deps = ["Random", "Serialization", "Sockets"]
+uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+
+[[deps.DocStringExtensions]]
+deps = ["LibGit2"]
+git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d"
+uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+version = "0.9.3"
+
+[[deps.Downloads]]
+deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+version = "1.6.0"
+
+[[deps.EnumX]]
+git-tree-sha1 = "bdb1942cd4c45e3c678fd11569d5cccd80976237"
+uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
+version = "1.0.4"
+
+[[deps.EnzymeCore]]
+git-tree-sha1 = "2c0192b96d5c45dbfb0f54e8cfb1256fece7b4ff"
+uuid = "f151be2c-9106-41f4-ab19-57ee4f262869"
+version = "0.7.1"
+weakdeps = ["Adapt"]
+
+    [deps.EnzymeCore.extensions]
+    AdaptExt = "Adapt"
+
+[[deps.ExprTools]]
+git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec"
+uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
+version = "0.1.10"
+
+[[deps.FastBroadcast]]
+deps = ["ArrayInterface", "LinearAlgebra", "Polyester", "Static", "StaticArrayInterface", "StrideArraysCore"]
+git-tree-sha1 = "a6e756a880fc419c8b41592010aebe6a5ce09136"
+uuid = "7034ab61-46d4-4ed7-9d0f-46aef9175898"
+version = "0.2.8"
+
+[[deps.FastClosures]]
+git-tree-sha1 = "acebe244d53ee1b461970f8910c235b259e772ef"
+uuid = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a"
+version = "0.3.2"
+
+[[deps.FileWatching]]
+uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
+
+[[deps.FiniteDiff]]
+deps = ["ArrayInterface", "LinearAlgebra", "Requires", "Setfield", "SparseArrays"]
+git-tree-sha1 = "bc0c5092d6caaea112d3c8e3b238d61563c58d5f"
+uuid = "6a86dc24-6348-571c-b903-95158fe2bd41"
+version = "2.23.0"
+
+    [deps.FiniteDiff.extensions]
+    FiniteDiffBandedMatricesExt = "BandedMatrices"
+    FiniteDiffBlockBandedMatricesExt = "BlockBandedMatrices"
+    FiniteDiffStaticArraysExt = "StaticArrays"
+
+    [deps.FiniteDiff.weakdeps]
+    BandedMatrices = "aae01518-5342-5314-be14-df237901396f"
+    BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0"
+    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+
+[[deps.ForwardDiff]]
+deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"]
+git-tree-sha1 = "cf0fe81336da9fb90944683b8c41984b08793dad"
+uuid = "f6369f11-7733-5829-9624-2563aa707210"
+version = "0.10.36"
+
+    [deps.ForwardDiff.extensions]
+    ForwardDiffStaticArraysExt = "StaticArrays"
+
+    [deps.ForwardDiff.weakdeps]
+    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+
+[[deps.FunctionWrappers]]
+git-tree-sha1 = "d62485945ce5ae9c0c48f124a84998d755bae00e"
+uuid = "069b7b12-0de2-55c6-9aab-29f3d0a68a2e"
+version = "1.1.3"
+
+[[deps.FunctionWrappersWrappers]]
+deps = ["FunctionWrappers"]
+git-tree-sha1 = "b104d487b34566608f8b4e1c39fb0b10aa279ff8"
+uuid = "77dc65aa-8811-40c2-897b-53d922fa7daf"
+version = "0.1.3"
+
+[[deps.Future]]
+deps = ["Random"]
+uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
+
+[[deps.GPUArraysCore]]
+deps = ["Adapt"]
+git-tree-sha1 = "ec632f177c0d990e64d955ccc1b8c04c485a0950"
+uuid = "46192b85-c4d5-4398-a991-12ede77f4527"
+version = "0.1.6"
+
+[[deps.IfElse]]
+git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1"
+uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173"
+version = "0.1.1"
+
+[[deps.InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[deps.InverseFunctions]]
+deps = ["Test"]
+git-tree-sha1 = "896385798a8d49a255c398bd49162062e4a4c435"
+uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
+version = "0.1.13"
+weakdeps = ["Dates"]
+
+    [deps.InverseFunctions.extensions]
+    DatesExt = "Dates"
+
+[[deps.IrrationalConstants]]
+git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2"
+uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
+version = "0.2.2"
+
+[[deps.IteratorInterfaceExtensions]]
+git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
+uuid = "82899510-4779-5014-852e-03e436cf321d"
+version = "1.0.0"
+
+[[deps.JLLWrappers]]
+deps = ["Artifacts", "Preferences"]
+git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca"
+uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
+version = "1.5.0"
+
+[[deps.LayoutPointers]]
+deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface"]
+git-tree-sha1 = "62edfee3211981241b57ff1cedf4d74d79519277"
+uuid = "10f19ff3-798f-405d-979b-55457f8fc047"
+version = "0.1.15"
+
+[[deps.LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+version = "0.6.4"
+
+[[deps.LibCURL_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
+version = "8.4.0+0"
+
+[[deps.LibGit2]]
+deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[deps.LibGit2_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
+uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
+version = "1.6.4+0"
+
+[[deps.LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+version = "1.11.0+1"
+
+[[deps.Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[deps.LinearAlgebra]]
+deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
+uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+
+[[deps.LogExpFunctions]]
+deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
+git-tree-sha1 = "18144f3e9cbe9b15b070288eef858f71b291ce37"
+uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
+version = "0.3.27"
+
+    [deps.LogExpFunctions.extensions]
+    LogExpFunctionsChainRulesCoreExt = "ChainRulesCore"
+    LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables"
+    LogExpFunctionsInverseFunctionsExt = "InverseFunctions"
+
+    [deps.LogExpFunctions.weakdeps]
+    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+    ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
+    InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
+
+[[deps.Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[deps.MacroTools]]
+deps = ["Markdown", "Random"]
+git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.5.13"
+
+[[deps.ManualMemory]]
+git-tree-sha1 = "bcaef4fc7a0cfe2cba636d84cda54b5e4e4ca3cd"
+uuid = "d125e4d3-2237-4719-b19c-fa641b8a4667"
+version = "0.1.8"
+
+[[deps.Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[deps.MaybeInplace]]
+deps = ["ArrayInterface", "LinearAlgebra", "MacroTools", "SparseArrays"]
+git-tree-sha1 = "b1f2f92feb0bc201e91c155ef575bcc7d9cc3526"
+uuid = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb"
+version = "0.1.2"
+
+[[deps.MbedTLS_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+version = "2.28.2+1"
+
+[[deps.MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+version = "2023.1.10"
+
+[[deps.MuladdMacro]]
+git-tree-sha1 = "cac9cc5499c25554cba55cd3c30543cff5ca4fab"
+uuid = "46d2c3a1-f734-5fdb-9937-b9b9aeba4221"
+version = "0.2.4"
+
+[[deps.NaNMath]]
+deps = ["OpenLibm_jll"]
+git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4"
+uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
+version = "1.0.2"
+
+[[deps.NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+version = "1.2.0"
+
+[[deps.OpenBLAS_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
+uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
+version = "0.3.23+4"
+
+[[deps.OpenLibm_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
+version = "0.8.1+2"
+
+[[deps.OpenSpecFun_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
+uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
+version = "0.5.5+0"
+
+[[deps.OrderedCollections]]
+git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5"
+uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+version = "1.6.3"
+
+[[deps.Parameters]]
+deps = ["OrderedCollections", "UnPack"]
+git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe"
+uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a"
+version = "0.12.3"
+
+[[deps.Pkg]]
+deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+version = "1.10.0"
+
+[[deps.Polyester]]
+deps = ["ArrayInterface", "BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "ManualMemory", "PolyesterWeave", "Requires", "Static", "StaticArrayInterface", "StrideArraysCore", "ThreadingUtilities"]
+git-tree-sha1 = "5d8a46101b622927a87fe3553ea697e606d9a3c5"
+uuid = "f517fe37-dbe3-4b94-8317-1923a5111588"
+version = "0.7.11"
+
+[[deps.PolyesterWeave]]
+deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"]
+git-tree-sha1 = "240d7170f5ffdb285f9427b92333c3463bf65bf6"
+uuid = "1d0040c9-8b98-4ee7-8388-3f51789ca0ad"
+version = "0.2.1"
+
+[[deps.PreallocationTools]]
+deps = ["Adapt", "ArrayInterface", "ForwardDiff"]
+git-tree-sha1 = "b6665214f2d0739f2d09a17474dd443b9139784a"
+uuid = "d236fae5-4411-538c-8e31-a6e3d9e00b46"
+version = "0.4.20"
+
+    [deps.PreallocationTools.extensions]
+    PreallocationToolsReverseDiffExt = "ReverseDiff"
+
+    [deps.PreallocationTools.weakdeps]
+    ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
+
+[[deps.PrecompileTools]]
+deps = ["Preferences"]
+git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
+uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+version = "1.2.1"
+
+[[deps.Preferences]]
+deps = ["TOML"]
+git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+version = "1.4.3"
+
+[[deps.Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[deps.REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[deps.Random]]
+deps = ["SHA"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[deps.RecipesBase]]
+deps = ["PrecompileTools"]
+git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff"
+uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
+version = "1.3.4"
+
+[[deps.RecursiveArrayTools]]
+deps = ["Adapt", "ArrayInterface", "DocStringExtensions", "GPUArraysCore", "IteratorInterfaceExtensions", "LinearAlgebra", "RecipesBase", "SparseArrays", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables"]
+git-tree-sha1 = "d8f131090f2e44b145084928856a561c83f43b27"
+uuid = "731186ca-8d62-57ce-b412-fbd966d074cd"
+version = "3.13.0"
+
+    [deps.RecursiveArrayTools.extensions]
+    RecursiveArrayToolsFastBroadcastExt = "FastBroadcast"
+    RecursiveArrayToolsForwardDiffExt = "ForwardDiff"
+    RecursiveArrayToolsMeasurementsExt = "Measurements"
+    RecursiveArrayToolsMonteCarloMeasurementsExt = "MonteCarloMeasurements"
+    RecursiveArrayToolsReverseDiffExt = ["ReverseDiff", "Zygote"]
+    RecursiveArrayToolsTrackerExt = "Tracker"
+    RecursiveArrayToolsZygoteExt = "Zygote"
+
+    [deps.RecursiveArrayTools.weakdeps]
+    FastBroadcast = "7034ab61-46d4-4ed7-9d0f-46aef9175898"
+    ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+    Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7"
+    MonteCarloMeasurements = "0987c9cc-fe09-11e8-30f0-b96dd679fdca"
+    ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
+    Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
+    Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
+
+[[deps.Reexport]]
+git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
+uuid = "189a3867-3050-52da-a836-e630ba90ab69"
+version = "1.2.2"
+
+[[deps.Requires]]
+deps = ["UUIDs"]
+git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "1.3.0"
+
+[[deps.RuntimeGeneratedFunctions]]
+deps = ["ExprTools", "SHA", "Serialization"]
+git-tree-sha1 = "6aacc5eefe8415f47b3e34214c1d79d2674a0ba2"
+uuid = "7e49a35a-f44a-4d26-94aa-eba1b4ca6b47"
+version = "0.5.12"
+
+[[deps.SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[deps.SIMDTypes]]
+git-tree-sha1 = "330289636fb8107c5f32088d2741e9fd7a061a5c"
+uuid = "94e857df-77ce-4151-89e5-788b33177be4"
+version = "0.1.0"
+
+[[deps.SciMLBase]]
+deps = ["ADTypes", "ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "PrecompileTools", "Preferences", "Printf", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLOperators", "SciMLStructures", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables"]
+git-tree-sha1 = "d15c65e25615272e1b1c5edb1d307484c7942824"
+uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+version = "2.31.0"
+
+    [deps.SciMLBase.extensions]
+    SciMLBaseChainRulesCoreExt = "ChainRulesCore"
+    SciMLBaseMakieExt = "Makie"
+    SciMLBasePartialFunctionsExt = "PartialFunctions"
+    SciMLBasePyCallExt = "PyCall"
+    SciMLBasePythonCallExt = "PythonCall"
+    SciMLBaseRCallExt = "RCall"
+    SciMLBaseZygoteExt = "Zygote"
+
+    [deps.SciMLBase.weakdeps]
+    ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2"
+    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+    Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a"
+    PartialFunctions = "570af359-4316-4cb7-8c74-252c00c2016b"
+    PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
+    PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
+    RCall = "6f49c342-dc21-5d91-9882-a32aef131414"
+    Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
+
+[[deps.SciMLOperators]]
+deps = ["ArrayInterface", "DocStringExtensions", "LinearAlgebra", "MacroTools", "Setfield", "SparseArrays", "StaticArraysCore"]
+git-tree-sha1 = "10499f619ef6e890f3f4a38914481cc868689cd5"
+uuid = "c0aeaf25-5076-4817-a8d5-81caf7dfa961"
+version = "0.3.8"
+
+[[deps.SciMLStructures]]
+git-tree-sha1 = "5833c10ce83d690c124beedfe5f621b50b02ba4d"
+uuid = "53ae85a6-f571-4167-b2af-e1d143709226"
+version = "1.1.0"
+
+[[deps.Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[deps.Setfield]]
+deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"]
+git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac"
+uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46"
+version = "1.1.1"
+
+[[deps.Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[deps.SparseArrays]]
+deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+version = "1.10.0"
+
+[[deps.SpecialFunctions]]
+deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
+git-tree-sha1 = "e2cfc4012a19088254b3950b85c3c1d8882d864d"
+uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
+version = "2.3.1"
+
+    [deps.SpecialFunctions.extensions]
+    SpecialFunctionsChainRulesCoreExt = "ChainRulesCore"
+
+    [deps.SpecialFunctions.weakdeps]
+    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+
+[[deps.Static]]
+deps = ["IfElse"]
+git-tree-sha1 = "d2fdac9ff3906e27f7a618d47b676941baa6c80c"
+uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3"
+version = "0.8.10"
+
+[[deps.StaticArrayInterface]]
+deps = ["ArrayInterface", "Compat", "IfElse", "LinearAlgebra", "PrecompileTools", "Requires", "SparseArrays", "Static", "SuiteSparse"]
+git-tree-sha1 = "5d66818a39bb04bf328e92bc933ec5b4ee88e436"
+uuid = "0d7ed370-da01-4f52-bd93-41d350b8b718"
+version = "1.5.0"
+
+    [deps.StaticArrayInterface.extensions]
+    StaticArrayInterfaceOffsetArraysExt = "OffsetArrays"
+    StaticArrayInterfaceStaticArraysExt = "StaticArrays"
+
+    [deps.StaticArrayInterface.weakdeps]
+    OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
+    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+
+[[deps.StaticArraysCore]]
+git-tree-sha1 = "36b3d696ce6366023a0ea192b4cd442268995a0d"
+uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
+version = "1.4.2"
+
+[[deps.Statistics]]
+deps = ["LinearAlgebra", "SparseArrays"]
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+version = "1.10.0"
+
+[[deps.StrideArraysCore]]
+deps = ["ArrayInterface", "CloseOpenIntervals", "IfElse", "LayoutPointers", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface", "ThreadingUtilities"]
+git-tree-sha1 = "d6415f66f3d89c615929af907fdc6a3e17af0d8c"
+uuid = "7792a7ef-975c-4747-a70f-980b88e8d1da"
+version = "0.5.2"
+
+[[deps.SuiteSparse]]
+deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
+uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"
+
+[[deps.SuiteSparse_jll]]
+deps = ["Artifacts", "Libdl", "libblastrampoline_jll"]
+uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c"
+version = "7.2.1+1"
+
+[[deps.SymbolicIndexingInterface]]
+deps = ["Accessors", "ArrayInterface", "MacroTools", "RuntimeGeneratedFunctions", "StaticArraysCore"]
+git-tree-sha1 = "4b7f4c80449d8baae8857d55535033981862619c"
+uuid = "2efcf032-c050-4f8e-a9bb-153293bab1f5"
+version = "0.3.15"
+
+[[deps.TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+version = "1.0.3"
+
+[[deps.TableTraits]]
+deps = ["IteratorInterfaceExtensions"]
+git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
+uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
+version = "1.0.1"
+
+[[deps.Tables]]
+deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits"]
+git-tree-sha1 = "cb76cf677714c095e535e3501ac7954732aeea2d"
+uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
+version = "1.11.1"
+
+[[deps.Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+version = "1.10.0"
+
+[[deps.Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[deps.ThreadingUtilities]]
+deps = ["ManualMemory"]
+git-tree-sha1 = "eda08f7e9818eb53661b3deb74e3159460dfbc27"
+uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5"
+version = "0.5.2"
+
+[[deps.Tricks]]
+git-tree-sha1 = "eae1bb484cd63b36999ee58be2de6c178105112f"
+uuid = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775"
+version = "0.1.8"
+
+[[deps.TruncatedStacktraces]]
+deps = ["InteractiveUtils", "MacroTools", "Preferences"]
+git-tree-sha1 = "ea3e54c2bdde39062abf5a9758a23735558705e1"
+uuid = "781d530d-4396-4725-bb49-402e4bee1e77"
+version = "1.4.0"
+
+[[deps.UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[deps.UnPack]]
+git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b"
+uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
+version = "1.0.2"
+
+[[deps.Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+
+[[deps.Zlib_jll]]
+deps = ["Libdl"]
+uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
+version = "1.2.13+1"
+
+[[deps.libblastrampoline_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
+version = "5.8.0+1"
+
+[[deps.nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+version = "1.52.0+1"
+
+[[deps.p7zip_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+version = "17.4.0+2"
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 225b36090..c1065039b 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.6.0"
+version = "1.7.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -43,7 +43,7 @@ ArrayInterface = "7.7"
 CUDA = "5.2"
 ChainRulesCore = "1.22"
 ConcreteStructs = "0.2.3"
-DiffEqBase = "6.146"
+DiffEqBase = "6.149"
 DiffResults = "1.1"
 FastClosures = "0.3"
 FiniteDiff = "2.22"
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index 15e544795..6fe121481 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -48,7 +48,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
     @bb δJ⁻¹n = copy(x)
     @bb δJ⁻¹ = copy(J⁻¹)
 
-    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
         termination_condition)
 
     ls_cache = __get_linesearch(alg) === Val(true) ?
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 856e31fd4..9f092648d 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -70,7 +70,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...
     τ_min = T(alg.τ_min)
     τ_max = T(alg.τ_max)
 
-    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
         termination_condition)
 
     fx_norm = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 462332280..934dc4763 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -34,7 +34,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
     T = eltype(x)
 
     autodiff = __get_concrete_autodiff(prob, alg.autodiff)
-    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
         termination_condition)
 
     @bb xo = copy(x)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index 680b9cd7c..c2c8b446f 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -13,7 +13,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
     T = eltype(x)
     fx = _get_fx(prob, x)
 
-    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
         termination_condition)
 
     @bb δx = copy(x)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 145a5467d..600892edd 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -61,7 +61,7 @@ end
 
     U, Vᵀ = __init_low_rank_jacobian(x, fx, x isa StaticArray ? threshold : Val(η))
 
-    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
         termination_condition)
 
     @bb xo = copy(x)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
index e84f59521..9735d0c8c 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
@@ -32,7 +32,7 @@ function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresPr
     @bb xo = copy(x)
     J, jac_cache = jacobian_cache(autodiff, prob.f, fx, x, prob.p)
 
-    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
         termination_condition)
 
     for i in 1:maxiters
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 03c4692e7..e6ccf6536 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -88,7 +88,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
     J, jac_cache = jacobian_cache(autodiff, prob.f, fx, x, prob.p)
     fx, ∇f = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
-    abstol, reltol, tc_cache = init_termination_cache(abstol, reltol, fx, x,
+    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
         termination_condition)
 
     # Set default trust region radius if not specified by user.
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 38be3437c..76e91fcb4 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -288,14 +288,30 @@ end
 # different. NonlinearSolve is more for robust / cached solvers while SimpleNonlinearSolve
 # is meant for low overhead solvers, users can opt into the other termination modes but the
 # default is to use the least overhead version.
-function init_termination_cache(abstol, reltol, du, u, ::Nothing)
-    return init_termination_cache(abstol, reltol, du, u, AbsNormTerminationMode())
+function init_termination_cache(prob::NonlinearProblem, abstol, reltol, du, u, ::Nothing)
+    return init_termination_cache(prob, abstol, reltol, du, u,
+        AbsNormTerminationMode(Base.Fix1(maximum, abs)))
 end
-function init_termination_cache(abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode)
+function init_termination_cache(
+        prob::NonlinearLeastSquaresProblem, abstol, reltol, du, u, ::Nothing)
+    return init_termination_cache(prob, abstol, reltol, du, u,
+        AbsNormTerminationMode(Base.Fix2(norm, 2)))
+end
+
+function init_termination_cache(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+        abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode)
     T = promote_type(eltype(du), eltype(u))
     abstol = __get_tolerance(u, abstol, T)
     reltol = __get_tolerance(u, reltol, T)
-    tc_cache = init(du, u, tc; abstol, reltol)
+    tc_ = if hasfield(typeof(tc), :internalnorm) && tc.internalnorm === nothing
+        internalnorm = ifelse(
+            prob isa NonlinearProblem, Base.Fix1(maximum, abs), Base.Fix2(norm, 2))
+        DiffEqBase.set_termination_mode_internalnorm(tc, internalnorm)
+    else
+        tc
+    end
+    tc_cache = init(du, u, tc_; abstol, reltol, use_deprecated_retcodes = Val(false))
     return DiffEqBase.get_abstol(tc_cache), DiffEqBase.get_reltol(tc_cache), tc_cache
 end
 
@@ -305,45 +321,25 @@ function check_termination(tc_cache, fx, x, xo, prob, alg)
 end
 function check_termination(tc_cache, fx, x, xo, prob, alg,
         ::AbstractNonlinearTerminationMode)
-    if Bool(tc_cache(fx, x, xo))
+    tc_cache(fx, x, xo) &&
         return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-    end
     return nothing
 end
 function check_termination(tc_cache, fx, x, xo, prob, alg,
         ::AbstractSafeNonlinearTerminationMode)
-    if Bool(tc_cache(fx, x, xo))
-        if tc_cache.retcode == NonlinearSafeTerminationReturnCode.Success
-            retcode = ReturnCode.Success
-        elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.PatienceTermination
-            retcode = ReturnCode.ConvergenceFailure
-        elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.ProtectiveTermination
-            retcode = ReturnCode.Unstable
-        else
-            error("Unknown termination code: $(tc_cache.retcode)")
-        end
-        return build_solution(prob, alg, x, fx; retcode)
-    end
+    tc_cache(fx, x, xo) &&
+        return build_solution(prob, alg, x, fx; retcode = tc_cache.retcode)
     return nothing
 end
 function check_termination(tc_cache, fx, x, xo, prob, alg,
         ::AbstractSafeBestNonlinearTerminationMode)
-    if Bool(tc_cache(fx, x, xo))
-        if tc_cache.retcode == NonlinearSafeTerminationReturnCode.Success
-            retcode = ReturnCode.Success
-        elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.PatienceTermination
-            retcode = ReturnCode.ConvergenceFailure
-        elseif tc_cache.retcode == NonlinearSafeTerminationReturnCode.ProtectiveTermination
-            retcode = ReturnCode.Unstable
-        else
-            error("Unknown termination code: $(tc_cache.retcode)")
-        end
+    if tc_cache(fx, x, xo)
         if isinplace(prob)
             prob.f(fx, x, prob.p)
         else
             fx = prob.f(x, prob.p)
         end
-        return build_solution(prob, alg, tc_cache.u, fx; retcode)
+        return build_solution(prob, alg, tc_cache.u, fx; retcode = tc_cache.retcode)
     end
     return nothing
 end

From 271156647463086af4c636fe798a12a5514e43c0 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sun, 31 Mar 2024 14:47:08 -0400
Subject: [PATCH 323/375] remove manifest

---
 lib/SimpleNonlinearSolve/Manifest.toml | 781 -------------------------
 lib/SimpleNonlinearSolve/Project.toml  |   6 +-
 2 files changed, 3 insertions(+), 784 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/Manifest.toml

diff --git a/lib/SimpleNonlinearSolve/Manifest.toml b/lib/SimpleNonlinearSolve/Manifest.toml
deleted file mode 100644
index a3a6bb313..000000000
--- a/lib/SimpleNonlinearSolve/Manifest.toml
+++ /dev/null
@@ -1,781 +0,0 @@
-# This file is machine-generated - editing it directly is not advised
-
-julia_version = "1.10.2"
-manifest_format = "2.0"
-project_hash = "fb617ddc9060eafbe5e2a53eddeb505916c65ad4"
-
-[[deps.ADTypes]]
-git-tree-sha1 = "016833eb52ba2d6bea9fcb50ca295980e728ee24"
-uuid = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
-version = "0.2.7"
-
-[[deps.Accessors]]
-deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "LinearAlgebra", "MacroTools", "Markdown", "Test"]
-git-tree-sha1 = "c0d491ef0b135fd7d63cbc6404286bc633329425"
-uuid = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
-version = "0.1.36"
-
-    [deps.Accessors.extensions]
-    AccessorsAxisKeysExt = "AxisKeys"
-    AccessorsIntervalSetsExt = "IntervalSets"
-    AccessorsStaticArraysExt = "StaticArrays"
-    AccessorsStructArraysExt = "StructArrays"
-    AccessorsUnitfulExt = "Unitful"
-
-    [deps.Accessors.weakdeps]
-    AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5"
-    IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
-    Requires = "ae029012-a4dd-5104-9daa-d747884805df"
-    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-    StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
-    Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
-
-[[deps.Adapt]]
-deps = ["LinearAlgebra", "Requires"]
-git-tree-sha1 = "6a55b747d1812e699320963ffde36f1ebdda4099"
-uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
-version = "4.0.4"
-
-    [deps.Adapt.extensions]
-    AdaptStaticArraysExt = "StaticArrays"
-
-    [deps.Adapt.weakdeps]
-    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-
-[[deps.ArgTools]]
-uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
-version = "1.1.1"
-
-[[deps.ArrayInterface]]
-deps = ["Adapt", "LinearAlgebra", "SparseArrays", "SuiteSparse"]
-git-tree-sha1 = "44691067188f6bd1b2289552a23e4b7572f4528d"
-uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
-version = "7.9.0"
-
-    [deps.ArrayInterface.extensions]
-    ArrayInterfaceBandedMatricesExt = "BandedMatrices"
-    ArrayInterfaceBlockBandedMatricesExt = "BlockBandedMatrices"
-    ArrayInterfaceCUDAExt = "CUDA"
-    ArrayInterfaceChainRulesExt = "ChainRules"
-    ArrayInterfaceGPUArraysCoreExt = "GPUArraysCore"
-    ArrayInterfaceReverseDiffExt = "ReverseDiff"
-    ArrayInterfaceStaticArraysCoreExt = "StaticArraysCore"
-    ArrayInterfaceTrackerExt = "Tracker"
-
-    [deps.ArrayInterface.weakdeps]
-    BandedMatrices = "aae01518-5342-5314-be14-df237901396f"
-    BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0"
-    CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
-    ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2"
-    GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527"
-    ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
-    StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
-    Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
-
-[[deps.Artifacts]]
-uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
-
-[[deps.Base64]]
-uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
-
-[[deps.BitTwiddlingConvenienceFunctions]]
-deps = ["Static"]
-git-tree-sha1 = "0c5f81f47bbbcf4aea7b2959135713459170798b"
-uuid = "62783981-4cbd-42fc-bca8-16325de8dc4b"
-version = "0.1.5"
-
-[[deps.CPUSummary]]
-deps = ["CpuId", "IfElse", "PrecompileTools", "Static"]
-git-tree-sha1 = "601f7e7b3d36f18790e2caf83a882d88e9b71ff1"
-uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9"
-version = "0.2.4"
-
-[[deps.CloseOpenIntervals]]
-deps = ["Static", "StaticArrayInterface"]
-git-tree-sha1 = "70232f82ffaab9dc52585e0dd043b5e0c6b714f1"
-uuid = "fb6a15b2-703c-40df-9091-08a04967cfa9"
-version = "0.1.12"
-
-[[deps.CommonSolve]]
-git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c"
-uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
-version = "0.2.4"
-
-[[deps.CommonSubexpressions]]
-deps = ["MacroTools", "Test"]
-git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7"
-uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
-version = "0.3.0"
-
-[[deps.Compat]]
-deps = ["TOML", "UUIDs"]
-git-tree-sha1 = "c955881e3c981181362ae4088b35995446298b80"
-uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "4.14.0"
-weakdeps = ["Dates", "LinearAlgebra"]
-
-    [deps.Compat.extensions]
-    CompatLinearAlgebraExt = "LinearAlgebra"
-
-[[deps.CompilerSupportLibraries_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
-version = "1.1.0+0"
-
-[[deps.CompositionsBase]]
-git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad"
-uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b"
-version = "0.1.2"
-weakdeps = ["InverseFunctions"]
-
-    [deps.CompositionsBase.extensions]
-    CompositionsBaseInverseFunctionsExt = "InverseFunctions"
-
-[[deps.ConcreteStructs]]
-git-tree-sha1 = "f749037478283d372048690eb3b5f92a79432b34"
-uuid = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
-version = "0.2.3"
-
-[[deps.ConstructionBase]]
-deps = ["LinearAlgebra"]
-git-tree-sha1 = "260fd2400ed2dab602a7c15cf10c1933c59930a2"
-uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
-version = "1.5.5"
-
-    [deps.ConstructionBase.extensions]
-    ConstructionBaseIntervalSetsExt = "IntervalSets"
-    ConstructionBaseStaticArraysExt = "StaticArrays"
-
-    [deps.ConstructionBase.weakdeps]
-    IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
-    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-
-[[deps.CpuId]]
-deps = ["Markdown"]
-git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406"
-uuid = "adafc99b-e345-5852-983c-f28acb93d879"
-version = "0.3.1"
-
-[[deps.DataAPI]]
-git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe"
-uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
-version = "1.16.0"
-
-[[deps.DataStructures]]
-deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "0f4b5d62a88d8f59003e43c25a8a90de9eb76317"
-uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.18.18"
-
-[[deps.DataValueInterfaces]]
-git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
-uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
-version = "1.0.0"
-
-[[deps.Dates]]
-deps = ["Printf"]
-uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
-
-[[deps.DiffEqBase]]
-deps = ["ArrayInterface", "ConcreteStructs", "DataStructures", "DocStringExtensions", "EnumX", "EnzymeCore", "FastBroadcast", "FastClosures", "ForwardDiff", "FunctionWrappers", "FunctionWrappersWrappers", "LinearAlgebra", "Logging", "Markdown", "MuladdMacro", "Parameters", "PreallocationTools", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Static", "StaticArraysCore", "Statistics", "Tricks", "TruncatedStacktraces"]
-git-tree-sha1 = "4fa023dbb15b3485426bbc6c43e030c14250d664"
-repo-rev = "ap/nlls"
-repo-url = "https://github.com/SciML/DiffEqBase.jl.git"
-uuid = "2b5f629d-d688-5b77-993f-72d75c75574e"
-version = "6.149.0"
-
-    [deps.DiffEqBase.extensions]
-    DiffEqBaseChainRulesCoreExt = "ChainRulesCore"
-    DiffEqBaseDistributionsExt = "Distributions"
-    DiffEqBaseEnzymeExt = ["ChainRulesCore", "Enzyme"]
-    DiffEqBaseGeneralizedGeneratedExt = "GeneralizedGenerated"
-    DiffEqBaseMPIExt = "MPI"
-    DiffEqBaseMeasurementsExt = "Measurements"
-    DiffEqBaseMonteCarloMeasurementsExt = "MonteCarloMeasurements"
-    DiffEqBaseReverseDiffExt = "ReverseDiff"
-    DiffEqBaseTrackerExt = "Tracker"
-    DiffEqBaseUnitfulExt = "Unitful"
-
-    [deps.DiffEqBase.weakdeps]
-    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-    Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
-    Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
-    GeneralizedGenerated = "6b9d7cbe-bcb9-11e9-073f-15a7a543e2eb"
-    MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
-    Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7"
-    MonteCarloMeasurements = "0987c9cc-fe09-11e8-30f0-b96dd679fdca"
-    ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
-    Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
-    Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
-
-[[deps.DiffResults]]
-deps = ["StaticArraysCore"]
-git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621"
-uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
-version = "1.1.0"
-
-[[deps.DiffRules]]
-deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"]
-git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272"
-uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
-version = "1.15.1"
-
-[[deps.Distributed]]
-deps = ["Random", "Serialization", "Sockets"]
-uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
-
-[[deps.DocStringExtensions]]
-deps = ["LibGit2"]
-git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d"
-uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
-version = "0.9.3"
-
-[[deps.Downloads]]
-deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
-uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
-version = "1.6.0"
-
-[[deps.EnumX]]
-git-tree-sha1 = "bdb1942cd4c45e3c678fd11569d5cccd80976237"
-uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
-version = "1.0.4"
-
-[[deps.EnzymeCore]]
-git-tree-sha1 = "2c0192b96d5c45dbfb0f54e8cfb1256fece7b4ff"
-uuid = "f151be2c-9106-41f4-ab19-57ee4f262869"
-version = "0.7.1"
-weakdeps = ["Adapt"]
-
-    [deps.EnzymeCore.extensions]
-    AdaptExt = "Adapt"
-
-[[deps.ExprTools]]
-git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec"
-uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
-version = "0.1.10"
-
-[[deps.FastBroadcast]]
-deps = ["ArrayInterface", "LinearAlgebra", "Polyester", "Static", "StaticArrayInterface", "StrideArraysCore"]
-git-tree-sha1 = "a6e756a880fc419c8b41592010aebe6a5ce09136"
-uuid = "7034ab61-46d4-4ed7-9d0f-46aef9175898"
-version = "0.2.8"
-
-[[deps.FastClosures]]
-git-tree-sha1 = "acebe244d53ee1b461970f8910c235b259e772ef"
-uuid = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a"
-version = "0.3.2"
-
-[[deps.FileWatching]]
-uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
-
-[[deps.FiniteDiff]]
-deps = ["ArrayInterface", "LinearAlgebra", "Requires", "Setfield", "SparseArrays"]
-git-tree-sha1 = "bc0c5092d6caaea112d3c8e3b238d61563c58d5f"
-uuid = "6a86dc24-6348-571c-b903-95158fe2bd41"
-version = "2.23.0"
-
-    [deps.FiniteDiff.extensions]
-    FiniteDiffBandedMatricesExt = "BandedMatrices"
-    FiniteDiffBlockBandedMatricesExt = "BlockBandedMatrices"
-    FiniteDiffStaticArraysExt = "StaticArrays"
-
-    [deps.FiniteDiff.weakdeps]
-    BandedMatrices = "aae01518-5342-5314-be14-df237901396f"
-    BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0"
-    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-
-[[deps.ForwardDiff]]
-deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"]
-git-tree-sha1 = "cf0fe81336da9fb90944683b8c41984b08793dad"
-uuid = "f6369f11-7733-5829-9624-2563aa707210"
-version = "0.10.36"
-
-    [deps.ForwardDiff.extensions]
-    ForwardDiffStaticArraysExt = "StaticArrays"
-
-    [deps.ForwardDiff.weakdeps]
-    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-
-[[deps.FunctionWrappers]]
-git-tree-sha1 = "d62485945ce5ae9c0c48f124a84998d755bae00e"
-uuid = "069b7b12-0de2-55c6-9aab-29f3d0a68a2e"
-version = "1.1.3"
-
-[[deps.FunctionWrappersWrappers]]
-deps = ["FunctionWrappers"]
-git-tree-sha1 = "b104d487b34566608f8b4e1c39fb0b10aa279ff8"
-uuid = "77dc65aa-8811-40c2-897b-53d922fa7daf"
-version = "0.1.3"
-
-[[deps.Future]]
-deps = ["Random"]
-uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
-
-[[deps.GPUArraysCore]]
-deps = ["Adapt"]
-git-tree-sha1 = "ec632f177c0d990e64d955ccc1b8c04c485a0950"
-uuid = "46192b85-c4d5-4398-a991-12ede77f4527"
-version = "0.1.6"
-
-[[deps.IfElse]]
-git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1"
-uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173"
-version = "0.1.1"
-
-[[deps.InteractiveUtils]]
-deps = ["Markdown"]
-uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
-
-[[deps.InverseFunctions]]
-deps = ["Test"]
-git-tree-sha1 = "896385798a8d49a255c398bd49162062e4a4c435"
-uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
-version = "0.1.13"
-weakdeps = ["Dates"]
-
-    [deps.InverseFunctions.extensions]
-    DatesExt = "Dates"
-
-[[deps.IrrationalConstants]]
-git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2"
-uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
-version = "0.2.2"
-
-[[deps.IteratorInterfaceExtensions]]
-git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
-uuid = "82899510-4779-5014-852e-03e436cf321d"
-version = "1.0.0"
-
-[[deps.JLLWrappers]]
-deps = ["Artifacts", "Preferences"]
-git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca"
-uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
-version = "1.5.0"
-
-[[deps.LayoutPointers]]
-deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface"]
-git-tree-sha1 = "62edfee3211981241b57ff1cedf4d74d79519277"
-uuid = "10f19ff3-798f-405d-979b-55457f8fc047"
-version = "0.1.15"
-
-[[deps.LibCURL]]
-deps = ["LibCURL_jll", "MozillaCACerts_jll"]
-uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
-version = "0.6.4"
-
-[[deps.LibCURL_jll]]
-deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
-uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
-version = "8.4.0+0"
-
-[[deps.LibGit2]]
-deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
-uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
-
-[[deps.LibGit2_jll]]
-deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
-uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
-version = "1.6.4+0"
-
-[[deps.LibSSH2_jll]]
-deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
-uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
-version = "1.11.0+1"
-
-[[deps.Libdl]]
-uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
-
-[[deps.LinearAlgebra]]
-deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
-uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-
-[[deps.LogExpFunctions]]
-deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
-git-tree-sha1 = "18144f3e9cbe9b15b070288eef858f71b291ce37"
-uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
-version = "0.3.27"
-
-    [deps.LogExpFunctions.extensions]
-    LogExpFunctionsChainRulesCoreExt = "ChainRulesCore"
-    LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables"
-    LogExpFunctionsInverseFunctionsExt = "InverseFunctions"
-
-    [deps.LogExpFunctions.weakdeps]
-    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-    ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
-    InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
-
-[[deps.Logging]]
-uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
-
-[[deps.MacroTools]]
-deps = ["Markdown", "Random"]
-git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df"
-uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
-version = "0.5.13"
-
-[[deps.ManualMemory]]
-git-tree-sha1 = "bcaef4fc7a0cfe2cba636d84cda54b5e4e4ca3cd"
-uuid = "d125e4d3-2237-4719-b19c-fa641b8a4667"
-version = "0.1.8"
-
-[[deps.Markdown]]
-deps = ["Base64"]
-uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
-
-[[deps.MaybeInplace]]
-deps = ["ArrayInterface", "LinearAlgebra", "MacroTools", "SparseArrays"]
-git-tree-sha1 = "b1f2f92feb0bc201e91c155ef575bcc7d9cc3526"
-uuid = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb"
-version = "0.1.2"
-
-[[deps.MbedTLS_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
-version = "2.28.2+1"
-
-[[deps.MozillaCACerts_jll]]
-uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
-version = "2023.1.10"
-
-[[deps.MuladdMacro]]
-git-tree-sha1 = "cac9cc5499c25554cba55cd3c30543cff5ca4fab"
-uuid = "46d2c3a1-f734-5fdb-9937-b9b9aeba4221"
-version = "0.2.4"
-
-[[deps.NaNMath]]
-deps = ["OpenLibm_jll"]
-git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4"
-uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
-version = "1.0.2"
-
-[[deps.NetworkOptions]]
-uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
-version = "1.2.0"
-
-[[deps.OpenBLAS_jll]]
-deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
-uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
-version = "0.3.23+4"
-
-[[deps.OpenLibm_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
-version = "0.8.1+2"
-
-[[deps.OpenSpecFun_jll]]
-deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
-uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
-version = "0.5.5+0"
-
-[[deps.OrderedCollections]]
-git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5"
-uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-version = "1.6.3"
-
-[[deps.Parameters]]
-deps = ["OrderedCollections", "UnPack"]
-git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe"
-uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a"
-version = "0.12.3"
-
-[[deps.Pkg]]
-deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
-uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-version = "1.10.0"
-
-[[deps.Polyester]]
-deps = ["ArrayInterface", "BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "ManualMemory", "PolyesterWeave", "Requires", "Static", "StaticArrayInterface", "StrideArraysCore", "ThreadingUtilities"]
-git-tree-sha1 = "5d8a46101b622927a87fe3553ea697e606d9a3c5"
-uuid = "f517fe37-dbe3-4b94-8317-1923a5111588"
-version = "0.7.11"
-
-[[deps.PolyesterWeave]]
-deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"]
-git-tree-sha1 = "240d7170f5ffdb285f9427b92333c3463bf65bf6"
-uuid = "1d0040c9-8b98-4ee7-8388-3f51789ca0ad"
-version = "0.2.1"
-
-[[deps.PreallocationTools]]
-deps = ["Adapt", "ArrayInterface", "ForwardDiff"]
-git-tree-sha1 = "b6665214f2d0739f2d09a17474dd443b9139784a"
-uuid = "d236fae5-4411-538c-8e31-a6e3d9e00b46"
-version = "0.4.20"
-
-    [deps.PreallocationTools.extensions]
-    PreallocationToolsReverseDiffExt = "ReverseDiff"
-
-    [deps.PreallocationTools.weakdeps]
-    ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
-
-[[deps.PrecompileTools]]
-deps = ["Preferences"]
-git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
-uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
-version = "1.2.1"
-
-[[deps.Preferences]]
-deps = ["TOML"]
-git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6"
-uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.4.3"
-
-[[deps.Printf]]
-deps = ["Unicode"]
-uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
-
-[[deps.REPL]]
-deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
-uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
-
-[[deps.Random]]
-deps = ["SHA"]
-uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-
-[[deps.RecipesBase]]
-deps = ["PrecompileTools"]
-git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff"
-uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
-version = "1.3.4"
-
-[[deps.RecursiveArrayTools]]
-deps = ["Adapt", "ArrayInterface", "DocStringExtensions", "GPUArraysCore", "IteratorInterfaceExtensions", "LinearAlgebra", "RecipesBase", "SparseArrays", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables"]
-git-tree-sha1 = "d8f131090f2e44b145084928856a561c83f43b27"
-uuid = "731186ca-8d62-57ce-b412-fbd966d074cd"
-version = "3.13.0"
-
-    [deps.RecursiveArrayTools.extensions]
-    RecursiveArrayToolsFastBroadcastExt = "FastBroadcast"
-    RecursiveArrayToolsForwardDiffExt = "ForwardDiff"
-    RecursiveArrayToolsMeasurementsExt = "Measurements"
-    RecursiveArrayToolsMonteCarloMeasurementsExt = "MonteCarloMeasurements"
-    RecursiveArrayToolsReverseDiffExt = ["ReverseDiff", "Zygote"]
-    RecursiveArrayToolsTrackerExt = "Tracker"
-    RecursiveArrayToolsZygoteExt = "Zygote"
-
-    [deps.RecursiveArrayTools.weakdeps]
-    FastBroadcast = "7034ab61-46d4-4ed7-9d0f-46aef9175898"
-    ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
-    Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7"
-    MonteCarloMeasurements = "0987c9cc-fe09-11e8-30f0-b96dd679fdca"
-    ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
-    Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
-    Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
-
-[[deps.Reexport]]
-git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
-uuid = "189a3867-3050-52da-a836-e630ba90ab69"
-version = "1.2.2"
-
-[[deps.Requires]]
-deps = ["UUIDs"]
-git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
-uuid = "ae029012-a4dd-5104-9daa-d747884805df"
-version = "1.3.0"
-
-[[deps.RuntimeGeneratedFunctions]]
-deps = ["ExprTools", "SHA", "Serialization"]
-git-tree-sha1 = "6aacc5eefe8415f47b3e34214c1d79d2674a0ba2"
-uuid = "7e49a35a-f44a-4d26-94aa-eba1b4ca6b47"
-version = "0.5.12"
-
-[[deps.SHA]]
-uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
-version = "0.7.0"
-
-[[deps.SIMDTypes]]
-git-tree-sha1 = "330289636fb8107c5f32088d2741e9fd7a061a5c"
-uuid = "94e857df-77ce-4151-89e5-788b33177be4"
-version = "0.1.0"
-
-[[deps.SciMLBase]]
-deps = ["ADTypes", "ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "PrecompileTools", "Preferences", "Printf", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLOperators", "SciMLStructures", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables"]
-git-tree-sha1 = "d15c65e25615272e1b1c5edb1d307484c7942824"
-uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
-version = "2.31.0"
-
-    [deps.SciMLBase.extensions]
-    SciMLBaseChainRulesCoreExt = "ChainRulesCore"
-    SciMLBaseMakieExt = "Makie"
-    SciMLBasePartialFunctionsExt = "PartialFunctions"
-    SciMLBasePyCallExt = "PyCall"
-    SciMLBasePythonCallExt = "PythonCall"
-    SciMLBaseRCallExt = "RCall"
-    SciMLBaseZygoteExt = "Zygote"
-
-    [deps.SciMLBase.weakdeps]
-    ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2"
-    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-    Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a"
-    PartialFunctions = "570af359-4316-4cb7-8c74-252c00c2016b"
-    PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
-    PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
-    RCall = "6f49c342-dc21-5d91-9882-a32aef131414"
-    Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
-
-[[deps.SciMLOperators]]
-deps = ["ArrayInterface", "DocStringExtensions", "LinearAlgebra", "MacroTools", "Setfield", "SparseArrays", "StaticArraysCore"]
-git-tree-sha1 = "10499f619ef6e890f3f4a38914481cc868689cd5"
-uuid = "c0aeaf25-5076-4817-a8d5-81caf7dfa961"
-version = "0.3.8"
-
-[[deps.SciMLStructures]]
-git-tree-sha1 = "5833c10ce83d690c124beedfe5f621b50b02ba4d"
-uuid = "53ae85a6-f571-4167-b2af-e1d143709226"
-version = "1.1.0"
-
-[[deps.Serialization]]
-uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
-
-[[deps.Setfield]]
-deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"]
-git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac"
-uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46"
-version = "1.1.1"
-
-[[deps.Sockets]]
-uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
-
-[[deps.SparseArrays]]
-deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"]
-uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
-version = "1.10.0"
-
-[[deps.SpecialFunctions]]
-deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
-git-tree-sha1 = "e2cfc4012a19088254b3950b85c3c1d8882d864d"
-uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
-version = "2.3.1"
-
-    [deps.SpecialFunctions.extensions]
-    SpecialFunctionsChainRulesCoreExt = "ChainRulesCore"
-
-    [deps.SpecialFunctions.weakdeps]
-    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-
-[[deps.Static]]
-deps = ["IfElse"]
-git-tree-sha1 = "d2fdac9ff3906e27f7a618d47b676941baa6c80c"
-uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3"
-version = "0.8.10"
-
-[[deps.StaticArrayInterface]]
-deps = ["ArrayInterface", "Compat", "IfElse", "LinearAlgebra", "PrecompileTools", "Requires", "SparseArrays", "Static", "SuiteSparse"]
-git-tree-sha1 = "5d66818a39bb04bf328e92bc933ec5b4ee88e436"
-uuid = "0d7ed370-da01-4f52-bd93-41d350b8b718"
-version = "1.5.0"
-
-    [deps.StaticArrayInterface.extensions]
-    StaticArrayInterfaceOffsetArraysExt = "OffsetArrays"
-    StaticArrayInterfaceStaticArraysExt = "StaticArrays"
-
-    [deps.StaticArrayInterface.weakdeps]
-    OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
-    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-
-[[deps.StaticArraysCore]]
-git-tree-sha1 = "36b3d696ce6366023a0ea192b4cd442268995a0d"
-uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
-version = "1.4.2"
-
-[[deps.Statistics]]
-deps = ["LinearAlgebra", "SparseArrays"]
-uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
-version = "1.10.0"
-
-[[deps.StrideArraysCore]]
-deps = ["ArrayInterface", "CloseOpenIntervals", "IfElse", "LayoutPointers", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface", "ThreadingUtilities"]
-git-tree-sha1 = "d6415f66f3d89c615929af907fdc6a3e17af0d8c"
-uuid = "7792a7ef-975c-4747-a70f-980b88e8d1da"
-version = "0.5.2"
-
-[[deps.SuiteSparse]]
-deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
-uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"
-
-[[deps.SuiteSparse_jll]]
-deps = ["Artifacts", "Libdl", "libblastrampoline_jll"]
-uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c"
-version = "7.2.1+1"
-
-[[deps.SymbolicIndexingInterface]]
-deps = ["Accessors", "ArrayInterface", "MacroTools", "RuntimeGeneratedFunctions", "StaticArraysCore"]
-git-tree-sha1 = "4b7f4c80449d8baae8857d55535033981862619c"
-uuid = "2efcf032-c050-4f8e-a9bb-153293bab1f5"
-version = "0.3.15"
-
-[[deps.TOML]]
-deps = ["Dates"]
-uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
-version = "1.0.3"
-
-[[deps.TableTraits]]
-deps = ["IteratorInterfaceExtensions"]
-git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
-uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
-version = "1.0.1"
-
-[[deps.Tables]]
-deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits"]
-git-tree-sha1 = "cb76cf677714c095e535e3501ac7954732aeea2d"
-uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
-version = "1.11.1"
-
-[[deps.Tar]]
-deps = ["ArgTools", "SHA"]
-uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
-version = "1.10.0"
-
-[[deps.Test]]
-deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
-uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[[deps.ThreadingUtilities]]
-deps = ["ManualMemory"]
-git-tree-sha1 = "eda08f7e9818eb53661b3deb74e3159460dfbc27"
-uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5"
-version = "0.5.2"
-
-[[deps.Tricks]]
-git-tree-sha1 = "eae1bb484cd63b36999ee58be2de6c178105112f"
-uuid = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775"
-version = "0.1.8"
-
-[[deps.TruncatedStacktraces]]
-deps = ["InteractiveUtils", "MacroTools", "Preferences"]
-git-tree-sha1 = "ea3e54c2bdde39062abf5a9758a23735558705e1"
-uuid = "781d530d-4396-4725-bb49-402e4bee1e77"
-version = "1.4.0"
-
-[[deps.UUIDs]]
-deps = ["Random", "SHA"]
-uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
-
-[[deps.UnPack]]
-git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b"
-uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
-version = "1.0.2"
-
-[[deps.Unicode]]
-uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
-
-[[deps.Zlib_jll]]
-deps = ["Libdl"]
-uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.13+1"
-
-[[deps.libblastrampoline_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
-version = "5.8.0+1"
-
-[[deps.nghttp2_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
-version = "1.52.0+1"
-
-[[deps.p7zip_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
-version = "17.4.0+2"
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index c1065039b..8d279f30e 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -39,13 +39,13 @@ SimpleNonlinearSolveZygoteExt = "Zygote"
 ADTypes = "0.2.6"
 AllocCheck = "0.1.1"
 Aqua = "0.8"
-ArrayInterface = "7.7"
+ArrayInterface = "7.8"
 CUDA = "5.2"
 ChainRulesCore = "1.22"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.149"
 DiffResults = "1.1"
-FastClosures = "0.3"
+FastClosures = "0.3.2"
 FiniteDiff = "2.22"
 ForwardDiff = "0.10.36"
 LinearAlgebra = "1.10"
@@ -59,7 +59,7 @@ Random = "1.10"
 ReTestItems = "1.23"
 Reexport = "1.2"
 ReverseDiff = "1.15"
-SciMLBase = "2.26.3"
+SciMLBase = "2.28.0"
 SciMLSensitivity = "7.56"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"

From 7f3cc72bbcf6b8aa0b8c25d6f0728e739beb457f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 1 Apr 2024 21:25:20 +0000
Subject: [PATCH 324/375] Bump julia-actions/setup-julia from 1 to 2

Bumps [julia-actions/setup-julia](https://github.com/julia-actions/setup-julia) from 1 to 2.
- [Release notes](https://github.com/julia-actions/setup-julia/releases)
- [Commits](https://github.com/julia-actions/setup-julia/compare/v1...v2)

---
updated-dependencies:
- dependency-name: julia-actions/setup-julia
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 lib/SimpleNonlinearSolve/.github/workflows/CI.yml            | 2 +-
 lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml     | 2 +-
 lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml    | 2 +-
 lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
index 16ec1087c..295de1024 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
@@ -22,7 +22,7 @@ jobs:
           - windows-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
       - uses: actions/cache@v4
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
index 0d225bb0b..a991ffbfd 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downgrade.yml
@@ -18,7 +18,7 @@ jobs:
         version: ['1']
     steps:
       - uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
       - uses: julia-actions/julia-downgrade-compat@v1
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
index a3c5003c1..9d509032f 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Downstream.yml
@@ -27,7 +27,7 @@ jobs:
           - {user: SciML, repo: NonlinearSolve.jl, group: All}
     steps:
       - uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.julia-version }}
           arch: x64
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml b/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
index 28b9ce2fa..66c86a362 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
@@ -16,7 +16,7 @@ jobs:
     if: github.base_ref == github.event.repository.default_branch
     runs-on: ubuntu-latest
     steps:
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: '1'
     - uses: actions/checkout@v4

From abf1a2fd718d75a20572211869e1d7e1aee6145b Mon Sep 17 00:00:00 2001
From: CompatHelper Julia <compathelper_noreply@julialang.org>
Date: Fri, 19 Apr 2024 22:04:48 +0000
Subject: [PATCH 325/375] CompatHelper: bump compat for ADTypes to 1, (keep
 existing compat)

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 8d279f30e..c601ac6b0 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -36,7 +36,7 @@ SimpleNonlinearSolveTrackerExt = "Tracker"
 SimpleNonlinearSolveZygoteExt = "Zygote"
 
 [compat]
-ADTypes = "0.2.6"
+ADTypes = "0.2.6, 1"
 AllocCheck = "0.1.1"
 Aqua = "0.8"
 ArrayInterface = "7.8"

From e3cf53a7355ac47c84dee0b8fdf7ecc1b57cfbac Mon Sep 17 00:00:00 2001
From: Vaibhav Kumar Dixit <vaibhavyashdixit@gmail.com>
Date: Sat, 4 May 2024 12:32:32 -0400
Subject: [PATCH 326/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index c601ac6b0..1787816ff 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.7.0"
+version = "1.8.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From f5240b60f649e744e286fb2e530ad82be067a65c Mon Sep 17 00:00:00 2001
From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com>
Date: Thu, 16 May 2024 08:13:27 +0200
Subject: [PATCH 327/375] Force ADTypes v1

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 1787816ff..5a64b5e62 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -36,7 +36,7 @@ SimpleNonlinearSolveTrackerExt = "Tracker"
 SimpleNonlinearSolveZygoteExt = "Zygote"
 
 [compat]
-ADTypes = "0.2.6, 1"
+ADTypes = "1"
 AllocCheck = "0.1.1"
 Aqua = "0.8"
 ArrayInterface = "7.8"

From 59f0d7dcabb4fb3c934e960ef45527f8e13faf48 Mon Sep 17 00:00:00 2001
From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com>
Date: Thu, 16 May 2024 11:56:51 +0200
Subject: [PATCH 328/375] Bump version

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 5a64b5e62..9a95cb7fd 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.8.0"
+version = "1.8.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From a9f92b60dfe849cfaed14f9e92d0adbbf1d7e07f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Fri, 17 May 2024 11:40:43 -0400
Subject: [PATCH 329/375] Clean up the testing

---
 .../.buildkite/pipeline.yml                   |  2 +-
 .../.github/workflows/FormatCheck.yml         | 43 ++---------------
 lib/SimpleNonlinearSolve/Project.toml         | 10 ++--
 .../test/core/23_test_problems_tests.jl       | 16 ++++---
 .../test/core/adjoint_tests.jl                |  2 +-
 .../test/core/aqua_tests.jl                   |  2 +-
 .../test/core/forward_ad_tests.jl             |  4 +-
 .../test/core/least_squares_tests.jl          |  2 +-
 .../test/core/matrix_resizing_tests.jl        |  2 +-
 .../test/core/rootfind_tests.jl               | 48 ++++---------------
 .../test/gpu/cuda_tests.jl                    |  4 +-
 lib/SimpleNonlinearSolve/test/runtests.jl     | 15 +++---
 12 files changed, 46 insertions(+), 104 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
index f95a8351f..6083fe1f0 100644
--- a/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
+++ b/lib/SimpleNonlinearSolve/.buildkite/pipeline.yml
@@ -22,4 +22,4 @@ env:
   JULIA_PKG_SERVER: ""
   RETESTITEMS_NWORKERS: 4
   RETESTITEMS_NWORKER_THREADS: 2
-  SECRET_CODECOV_TOKEN: "HsC3X/h3CrsRhMR199BUbw9xmASfTTEtdT2JMJlbkW6nsMtZXDIWxKzBNOpn0123zn3K+VE30i7+aLpcqZjlux0BQDMVLx5O5KudjrJmBE2G8KMMv1pra+UHEOFdFP3TzjHXGcHzpUrjpG8if8cHbsWdyQCV0Hdx2qAPnhX5haGPyMuIlRfoaWQfdxJPl5fDLXs1xe0LnMcYMx8uUsvvJZ/hhFtMYWJU0WFtqnAkCR8h/pQd6yZOaGP2SXJkOR8+1xbx+M8m6agH9idp2BjDPpXMOo27V3O2Gkoy3R4b5ouLdqOMaZSIOemoYCv6oh+EkbKaFvZydvm0xCW5YBFPPw==;U2FsdGVkX19FT0yFV6EMY/NVSQslXE6byckN0qa/HVU5dd3d4swmOCWBkBPtemRPGvCMP669iXSPfDTlw7ZSvw=="
\ No newline at end of file
+  SECRET_CODECOV_TOKEN: "HsC3X/h3CrsRhMR199BUbw9xmASfTTEtdT2JMJlbkW6nsMtZXDIWxKzBNOpn0123zn3K+VE30i7+aLpcqZjlux0BQDMVLx5O5KudjrJmBE2G8KMMv1pra+UHEOFdFP3TzjHXGcHzpUrjpG8if8cHbsWdyQCV0Hdx2qAPnhX5haGPyMuIlRfoaWQfdxJPl5fDLXs1xe0LnMcYMx8uUsvvJZ/hhFtMYWJU0WFtqnAkCR8h/pQd6yZOaGP2SXJkOR8+1xbx+M8m6agH9idp2BjDPpXMOo27V3O2Gkoy3R4b5ouLdqOMaZSIOemoYCv6oh+EkbKaFvZydvm0xCW5YBFPPw==;U2FsdGVkX19FT0yFV6EMY/NVSQslXE6byckN0qa/HVU5dd3d4swmOCWBkBPtemRPGvCMP669iXSPfDTlw7ZSvw=="
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
index 45bd09c47..8601ad558 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
@@ -1,42 +1,9 @@
-name: format-check
+name: Format suggestions
 
-on:
-  push:
-    branches:
-      - 'main'
-      - 'release-'
-    tags: '*'
-  pull_request:
+on: [pull_request]
 
 jobs:
-  build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        julia-version: [1]
-        julia-arch: [x86]
-        os: [ubuntu-latest]
+  code-style:
+    runs-on: ubuntu-latest
     steps:
-      - uses: julia-actions/setup-julia@latest
-        with:
-          version: ${{ matrix.julia-version }}
-
-      - uses: actions/checkout@v4
-      - name: Install JuliaFormatter and format
-        # This will use the latest version by default but you can set the version like so:
-        #
-        # julia  -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter", version="0.13.0"))'
-        run: |
-          julia  -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter"))'
-          julia  -e 'using JuliaFormatter; format(".", verbose=true)'
-      - name: Format check
-        run: |
-          julia -e '
-          out = Cmd(`git diff --name-only`) |> read |> String
-          if out == ""
-              exit(0)
-          else
-              @error "Some files have not been formatted !!!"
-              write(stdout, out)
-              exit(1)
-          end'
+      - uses: julia-actions/julia-format@v2
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 9a95cb7fd..322178ba9 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -36,10 +36,10 @@ SimpleNonlinearSolveTrackerExt = "Tracker"
 SimpleNonlinearSolveZygoteExt = "Zygote"
 
 [compat]
-ADTypes = "1"
+ADTypes = "1.2"
 AllocCheck = "0.1.1"
 Aqua = "0.8"
-ArrayInterface = "7.8"
+ArrayInterface = "7.9"
 CUDA = "5.2"
 ChainRulesCore = "1.22"
 ConcreteStructs = "0.2.3"
@@ -49,7 +49,7 @@ FastClosures = "0.3.2"
 FiniteDiff = "2.22"
 ForwardDiff = "0.10.36"
 LinearAlgebra = "1.10"
-LinearSolve = "2.25"
+LinearSolve = "2.30"
 MaybeInplace = "0.1.1"
 NonlinearProblemLibrary = "0.1.2"
 Pkg = "1.10"
@@ -59,8 +59,8 @@ Random = "1.10"
 ReTestItems = "1.23"
 Reexport = "1.2"
 ReverseDiff = "1.15"
-SciMLBase = "2.28.0"
-SciMLSensitivity = "7.56"
+SciMLBase = "2.37.0"
+SciMLSensitivity = "7.58"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"
 Test = "1.10"
diff --git a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
index 8b8f2395d..2180943fc 100644
--- a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
@@ -41,7 +41,7 @@ end
 export problems, dicts, test_on_library
 end
 
-@testitem "SimpleNewtonRaphson" setup=[RobustnessTesting] begin
+@testitem "SimpleNewtonRaphson" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleNewtonRaphson(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -50,7 +50,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "SimpleTrustRegion" setup=[RobustnessTesting] begin
+@testitem "SimpleTrustRegion" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleTrustRegion(),
         SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
 
@@ -61,16 +61,20 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "SimpleDFSane" setup=[RobustnessTesting] begin
+@testitem "SimpleDFSane" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleDFSane(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 11, 21]
+    if Sys.isapple()
+        broken_tests[alg_ops[1]] = [1, 2, 3, 5, 6, 21]
+    else
+        broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 11, 21]
+    end
 
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "SimpleBroyden" retries=5 setup=[RobustnessTesting] begin
+@testitem "SimpleBroyden" retries=5 setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleBroyden(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -79,7 +83,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "SimpleKlement" setup=[RobustnessTesting] begin
+@testitem "SimpleKlement" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleKlement(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
diff --git a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
index 74d745f65..749be1ada 100644
--- a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Simple Adjoint Test" begin
+@testitem "Simple Adjoint Test" tags=[:core] begin
     using ForwardDiff, ReverseDiff, SciMLSensitivity, Tracker, Zygote
 
     ff(u, p) = u .^ 2 .- p
diff --git a/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl b/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl
index 72437ff37..364f51b59 100644
--- a/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Aqua" begin
+@testitem "Aqua" tags=[:core] begin
     using Aqua
 
     Aqua.test_all(SimpleNonlinearSolve; piracies = false, ambiguities = false)
diff --git a/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
index e0852ba53..50fc18e94 100644
--- a/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
@@ -40,7 +40,7 @@ __compatible(::SimpleHalley, ::Val{:iip}) = false
 export test_f, test_f!, jacobian_f, solve_with, __compatible
 end
 
-@testitem "ForwardDiff.jl Integration: Rootfinding" setup=[ForwardADRootfindingTesting] begin
+@testitem "ForwardDiff.jl Integration: Rootfinding" setup=[ForwardADRootfindingTesting] tags=[:core] begin
     @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
         SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
         SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleDFSane())
@@ -133,7 +133,7 @@ export loss_function, loss_function!, loss_function_jac, loss_function_vjp,
        loss_function_jac!, loss_function_vjp!, θ_init, x, y_target
 end
 
-@testitem "ForwardDiff.jl Integration: NLLS" setup=[ForwardADNLLSTesting] begin
+@testitem "ForwardDiff.jl Integration: NLLS" setup=[ForwardADNLLSTesting] tags=[:core] begin
     @testset "$(nameof(typeof(alg)))" for alg in (
         SimpleNewtonRaphson(), SimpleGaussNewton(),
         SimpleNewtonRaphson(AutoFiniteDiff()), SimpleGaussNewton(AutoFiniteDiff()))
diff --git a/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl b/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
index 840a4f277..ef3a05504 100644
--- a/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Nonlinear Least Squares" begin
+@testitem "Nonlinear Least Squares" tags=[:core] begin
     using LinearAlgebra
 
     true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
diff --git a/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
index 54cf86b21..17cd3d674 100644
--- a/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Matrix Resizing" begin
+@testitem "Matrix Resizing" tags=[:core] begin
     ff(u, p) = u .* u .- p
     u0 = ones(2, 3)
     p = 2.0
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index 726a6dda7..ca0e26ef6 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -39,7 +39,7 @@ export quadratic_f, quadratic_f!, quadratic_f2, newton_fails, TERMINATION_CONDIT
 
 end
 
-@testitem "First Order Methods" setup=[RootfindingTesting] begin
+@testitem "First Order Methods" setup=[RootfindingTesting] tags=[:core] begin
     @testset "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion,
         (args...; kwargs...) -> SimpleTrustRegion(args...; nlsolve_update_rule = Val(true),
             kwargs...))
@@ -70,7 +70,7 @@ end
     end
 end
 
-@testitem "SimpleHalley" setup=[RootfindingTesting] begin
+@testitem "SimpleHalley" setup=[RootfindingTesting] tags=[:core] begin
     @testset "AutoDiff: $(nameof(typeof(autodiff)))" for autodiff in (AutoFiniteDiff(),
         AutoForwardDiff())
         @testset "[OOP] u0: $(nameof(typeof(u0)))" for u0 in ([1.0, 1.0],
@@ -89,7 +89,7 @@ end
     end
 end
 
-@testitem "Derivative Free Metods" setup=[RootfindingTesting] begin
+@testitem "Derivative Free Metods" setup=[RootfindingTesting] tags=[:core] begin
     @testset "$(nameof(typeof(alg)))" for alg in [SimpleBroyden(), SimpleKlement(),
         SimpleDFSane(), SimpleLimitedMemoryBroyden(),
         SimpleBroyden(; linesearch = Val(true)),
@@ -115,7 +115,7 @@ end
     end
 end
 
-@testitem "Newton Fails" setup=[RootfindingTesting] begin
+@testitem "Newton Fails" setup=[RootfindingTesting] tags=[:core] begin
     u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
     p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
 
@@ -127,13 +127,13 @@ end
     end
 end
 
-@testitem "Kwargs Propagation" setup=[RootfindingTesting] begin
+@testitem "Kwargs Propagation" setup=[RootfindingTesting] tags=[:core] begin
     prob = NonlinearProblem(quadratic_f, ones(4), 2.0; maxiters = 2)
     sol = solve(prob, SimpleNewtonRaphson())
     @test sol.retcode === ReturnCode.MaxIters
 end
 
-@testitem "Allocation Checks" setup=[RootfindingTesting] begin
+@testitem "Allocation Checks" setup=[RootfindingTesting] tags=[:core] begin
     if Sys.islinux()  # Very slow on other OS
         @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
             SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
@@ -165,7 +165,7 @@ end
     end
 end
 
-@testitem "Interval Nonlinear Problems" setup=[RootfindingTesting] begin
+@testitem "Interval Nonlinear Problems" setup=[RootfindingTesting] tags=[:core] begin
     @testset "$(nameof(typeof(alg)))" for alg in (Bisection(), Falsi(), Ridder(), Brent(),
         ITP(), Alefeld())
         tspan = (1.0, 20.0)
@@ -209,7 +209,7 @@ end
     end
 end
 
-@testitem "Tolerance Tests Interval Methods" setup=[RootfindingTesting] begin
+@testitem "Tolerance Tests Interval Methods" setup=[RootfindingTesting] tags=[:core] begin
     @testset "$(nameof(typeof(alg)))" for alg in (Bisection(), Falsi(), ITP())
         tspan = (1.0, 20.0)
         probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
@@ -224,7 +224,7 @@ end
     end
 end
 
-@testitem "Tolerance Tests Interval Methods 2" setup=[RootfindingTesting] begin
+@testitem "Tolerance Tests Interval Methods 2" setup=[RootfindingTesting] tags=[:core] begin
     @testset "$(nameof(typeof(alg)))" for alg in (Ridder(), Brent())
         tspan = (1.0, 20.0)
         probB = IntervalNonlinearProblem(quadratic_f, tspan, 2.0)
@@ -239,7 +239,7 @@ end
     end
 end
 
-@testitem "Flipped Signs and Reversed Tspan" setup=[RootfindingTesting] begin
+@testitem "Flipped Signs and Reversed Tspan" setup=[RootfindingTesting] tags=[:core] begin
     @testset "$(nameof(typeof(alg)))" for alg in (Alefeld(), Bisection(), Falsi(), Brent(),
         ITP(), Ridder())
         f1(u, p) = u * u - p
@@ -257,31 +257,3 @@ end
         end
     end
 end
-
-# The following tests were included in the previos versions but these kwargs never did
-# anything!
-# # Garuntee Tests for Bisection
-# f = function (u, p)
-#     if u < 2.0
-#         return u - 2.0
-#     elseif u > 3.0
-#         return u - 3.0
-#     else
-#         return 0.0
-#     end
-# end
-# probB = IntervalNonlinearProblem(f, (0.0, 4.0))
-
-# sol = solve(probB, Bisection(; exact_left = true))
-# @test f(sol.left, nothing) < 0.0
-# @test f(nextfloat(sol.left), nothing) >= 0.0
-
-# sol = solve(probB, Bisection(; exact_right = true))
-# @test f(sol.right, nothing) >= 0.0
-# @test f(prevfloat(sol.right), nothing) <= 0.0
-
-# sol = solve(probB, Bisection(; exact_left = true, exact_right = true); immutable = false)
-# @test f(sol.left, nothing) < 0.0
-# @test f(nextfloat(sol.left), nothing) >= 0.0
-# @test f(sol.right, nothing) >= 0.0
-# @test f(prevfloat(sol.right), nothing) <= 0.0
diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
index 37999dada..dae6a87d7 100644
--- a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Solving on GPUs" begin
+@testitem "Solving on GPUs" tags=[:cuda] begin
     using StaticArrays, CUDA
 
     CUDA.allowscalar(false)
@@ -36,7 +36,7 @@
     end
 end
 
-@testitem "CUDA Kernel Launch Test" begin
+@testitem "CUDA Kernel Launch Test" tags=[:cuda] begin
     using StaticArrays, CUDA
 
     CUDA.allowscalar(false)
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 1a2e93bb1..2f81c8401 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -1,11 +1,10 @@
-using ReTestItems
+using ReTestItems, CUDA
 
-const GROUP = get(ENV, "GROUP", "All")
+const GROUP = get(ENV, "GROUP", CUDA.functional() ? "All" : "Core")
 
-if GROUP == "All" || GROUP == "Core"
-    ReTestItems.runtests(joinpath(@__DIR__, "core/"))
-end
-
-if GROUP == "GPU"
-    ReTestItems.runtests(joinpath(@__DIR__, "gpu/"))
+if GROUP == "All"
+    ReTestItems.runtests(@__DIR__)
+else
+    tags = [Symbol(lowercase(GROUP))]
+    ReTestItems.runtests(@__DIR__; tags)
 end

From 2bb3bbb50150ae1a67c50d8405f6aa394d5c4532 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 May 2024 21:10:06 +0000
Subject: [PATCH 330/375] --- updated-dependencies: - dependency-name:
 julia-actions/julia-format   dependency-type: direct:production  
 update-type: version-update:semver-major ...

Signed-off-by: dependabot[bot] <support@github.com>
---
 lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
index 8601ad558..0ddeb4ed1 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
@@ -6,4 +6,4 @@ jobs:
   code-style:
     runs-on: ubuntu-latest
     steps:
-      - uses: julia-actions/julia-format@v2
+      - uses: julia-actions/julia-format@v3

From 2dc8a7e3d5ea3e779cc9b82756b521bbf835a0db Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 12:49:49 -0700
Subject: [PATCH 331/375] Support BigFloat

---
 lib/SimpleNonlinearSolve/Project.toml         |  4 +--
 .../test/core/exotic_type_tests.jl            | 31 +++++++++++++++++++
 .../test/gpu/cuda_tests.jl                    |  4 +--
 3 files changed, 35 insertions(+), 4 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 322178ba9..74b6cde87 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.8.1"
+version = "1.8.2"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -50,7 +50,7 @@ FiniteDiff = "2.22"
 ForwardDiff = "0.10.36"
 LinearAlgebra = "1.10"
 LinearSolve = "2.30"
-MaybeInplace = "0.1.1"
+MaybeInplace = "0.1.3"
 NonlinearProblemLibrary = "0.1.2"
 Pkg = "1.10"
 PolyesterForwardDiff = "0.1.1"
diff --git a/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl
new file mode 100644
index 000000000..fff77a3d4
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl
@@ -0,0 +1,31 @@
+# File for different types of exotic types
+@testsetup module SimpleNonlinearSolveExoticTypeTests
+using SimpleNonlinearSolve
+
+fn_iip = NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p)
+fn_oop = NonlinearFunction{false}((u, p) -> u .* u .- p)
+
+u0 = BigFloat[1.0, 1.0, 1.0]
+prob_iip_bf = NonlinearProblem{true}(fn_iip, u0, BigFloat(2))
+prob_oop_bf = NonlinearProblem{false}(fn_oop, u0, BigFloat(2))
+
+export fn_iip, fn_oop, u0, prob_iip_bf, prob_oop_bf
+end
+
+@testitem "BigFloat Support" tags=[:core] setup=[SimpleNonlinearSolveExoticTypeTests] begin
+    using SimpleNonlinearSolve, LinearAlgebra
+
+    for alg in [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
+        SimpleTrustRegion(), SimpleLimitedMemoryBroyden(; threshold = 2),
+        SimpleHalley()]
+        sol = solve(prob_oop_bf, alg)
+        @test norm(sol.resid, Inf) < 1e-6
+        @test SciMLBase.successful_retcode(sol.retcode)
+
+        alg isa SimpleHalley && continue
+
+        sol = solve(prob_iip_bf, alg)
+        @test norm(sol.resid, Inf) < 1e-6
+        @test SciMLBase.successful_retcode(sol.retcode)
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
index dae6a87d7..9d31114d0 100644
--- a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Solving on GPUs" tags=[:cuda] begin
+@testitem "Solving on GPUs" tags=[:cuda] skip=:(using CUDA; !CUDA.functional()) begin
     using StaticArrays, CUDA
 
     CUDA.allowscalar(false)
@@ -36,7 +36,7 @@
     end
 end
 
-@testitem "CUDA Kernel Launch Test" tags=[:cuda] begin
+@testitem "CUDA Kernel Launch Test" tags=[:cuda] skip=:(using CUDA; !CUDA.functional()) begin
     using StaticArrays, CUDA
 
     CUDA.allowscalar(false)

From aeef552032b7c86fae15b6dd895baed9b846eff7 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 14:06:00 -0700
Subject: [PATCH 332/375] Increase timeout for Kernel Launch tests

---
 lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
index 9d31114d0..39ac422a4 100644
--- a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
@@ -36,7 +36,7 @@
     end
 end
 
-@testitem "CUDA Kernel Launch Test" tags=[:cuda] skip=:(using CUDA; !CUDA.functional()) begin
+@testitem "CUDA Kernel Launch Test" tags=[:cuda] skip=:(using CUDA; !CUDA.functional()) timeout=3600 begin
     using StaticArrays, CUDA
 
     CUDA.allowscalar(false)

From bfc439bf84c229dddea7fb63698eec631728f5b7 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 14:26:47 -0700
Subject: [PATCH 333/375] Make explicit imports for extensions

---
 .../SimpleNonlinearSolveChainRulesCoreExt.jl  | 11 ++--
 ...leNonlinearSolvePolyesterForwardDiffExt.jl |  3 +-
 .../ext/SimpleNonlinearSolveReverseDiffExt.jl | 62 +++++++++++--------
 .../SimpleNonlinearSolveStaticArraysExt.jl    |  2 +-
 .../ext/SimpleNonlinearSolveTrackerExt.jl     | 18 ++++--
 .../ext/SimpleNonlinearSolveZygoteExt.jl      |  3 +-
 .../src/SimpleNonlinearSolve.jl               |  6 +-
 lib/SimpleNonlinearSolve/src/utils.jl         | 10 +--
 8 files changed, 66 insertions(+), 49 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
index dc84cb3e8..814af8f39 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
@@ -1,14 +1,17 @@
 module SimpleNonlinearSolveChainRulesCoreExt
 
-using ChainRulesCore, DiffEqBase, SciMLBase, SimpleNonlinearSolve
+using ChainRulesCore: ChainRulesCore, NoTangent
+using DiffEqBase: DiffEqBase
+using SciMLBase: ChainRulesOriginator, NonlinearProblem, NonlinearLeastSquaresProblem
+using SimpleNonlinearSolve: SimpleNonlinearSolve
 
 # The expectation here is that no-one is using this directly inside a GPU kernel. We can
 # eventually lift this requirement using a custom adjoint
 function ChainRulesCore.rrule(::typeof(SimpleNonlinearSolve.__internal_solve_up),
-        prob::NonlinearProblem, sensealg, u0, u0_changed, p, p_changed, alg, args...;
-        kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
     out, ∇internal = DiffEqBase._solve_adjoint(prob, sensealg, u0, p,
-        SciMLBase.ChainRulesOriginator(), alg, args...; kwargs...)
+        ChainRulesOriginator(), alg, args...; kwargs...)
     function ∇__internal_solve_up(Δ)
         ∂f, ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
         return (∂f, ∂prob, ∂sensealg, ∂u0, NoTangent(), ∂p, NoTangent(), NoTangent(),
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
index 81cee481d..aa38d1c4e 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
@@ -1,6 +1,7 @@
 module SimpleNonlinearSolvePolyesterForwardDiffExt
 
-using SimpleNonlinearSolve, PolyesterForwardDiff
+using PolyesterForwardDiff: PolyesterForwardDiff
+using SimpleNonlinearSolve: SimpleNonlinearSolve
 
 @inline SimpleNonlinearSolve.__is_extension_loaded(::Val{:PolyesterForwardDiff}) = true
 
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
index e0bbda27e..f9bfe0965 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
@@ -1,60 +1,68 @@
 module SimpleNonlinearSolveReverseDiffExt
 
-using ArrayInterface, DiffEqBase, ReverseDiff, SciMLBase, SimpleNonlinearSolve
-import ReverseDiff: TrackedArray, TrackedReal
-import SimpleNonlinearSolve: __internal_solve_up
+using ArrayInterface: ArrayInterface
+using DiffEqBase: DiffEqBase
+using ReverseDiff: ReverseDiff, TrackedArray, TrackedReal
+using SciMLBase: ReverseDiffOriginator, NonlinearProblem, NonlinearLeastSquaresProblem
+using SimpleNonlinearSolve: SimpleNonlinearSolve
 
-function __internal_solve_up(
-        prob::NonlinearProblem, sensealg, u0::TrackedArray, u0_changed,
-        p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+function SimpleNonlinearSolve.__internal_solve_up(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
+        u0::TrackedArray, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0,
         u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
-function __internal_solve_up(
-        prob::NonlinearProblem, sensealg, u0, u0_changed,
+function SimpleNonlinearSolve.__internal_solve_up(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0, u0_changed,
         p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0,
         u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
-function __internal_solve_up(
-        prob::NonlinearProblem, sensealg, u0::TrackedArray, u0_changed,
-        p, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+function SimpleNonlinearSolve.__internal_solve_up(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
+        u0::TrackedArray, u0_changed, p, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0,
         u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
-function __internal_solve_up(prob::NonlinearProblem, sensealg,
+function SimpleNonlinearSolve.__internal_solve_up(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
         u0::AbstractArray{<:TrackedReal}, u0_changed, p::AbstractArray{<:TrackedReal},
         p_changed, alg, args...; kwargs...)
-    return __internal_solve_up(
+    return SimpleNonlinearSolve.__internal_solve_up(
         prob, sensealg, ArrayInterface.aos_to_soa(u0), true,
         ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
 end
 
-function __internal_solve_up(prob::NonlinearProblem, sensealg, u0, u0_changed,
+function SimpleNonlinearSolve.__internal_solve_up(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0, u0_changed,
         p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
-    return __internal_solve_up(
-        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
+    return SimpleNonlinearSolve.__internal_solve_up(
+        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p), true, alg, args...;
+        kwargs...)
 end
 
-function __internal_solve_up(prob::NonlinearProblem, sensealg,
+function SimpleNonlinearSolve.__internal_solve_up(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
         u0::AbstractArray{<:TrackedReal}, u0_changed, p, p_changed, alg, args...; kwargs...)
-    return __internal_solve_up(
-        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
+    return SimpleNonlinearSolve.__internal_solve_up(
+        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p), true, alg, args...;
+        kwargs...)
 end
 
-ReverseDiff.@grad function __internal_solve_up(
-        prob::NonlinearProblem, sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+ReverseDiff.@grad function SimpleNonlinearSolve.__internal_solve_up(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
     out, ∇internal = DiffEqBase._solve_adjoint(
         prob, sensealg, ReverseDiff.value(u0), ReverseDiff.value(p),
-        SciMLBase.ReverseDiffOriginator(), alg, args...; kwargs...)
-    function ∇__internal_solve_up(_args...)
+        ReverseDiffOriginator(), alg, args...; kwargs...)
+    function ∇SimpleNonlinearSolve.__internal_solve_up(_args...)
         ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(_args...)
         return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
     end
-    return Array(out), ∇__internal_solve_up
+    return Array(out), ∇SimpleNonlinearSolve.__internal_solve_up
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl
index 90318a82a..c865084ce 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl
@@ -1,6 +1,6 @@
 module SimpleNonlinearSolveStaticArraysExt
 
-using SimpleNonlinearSolve
+using SimpleNonlinearSolve: SimpleNonlinearSolve
 
 @inline SimpleNonlinearSolve.__is_extension_loaded(::Val{:StaticArrays}) = true
 
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
index 61ce14645..7b35de09b 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
@@ -1,8 +1,12 @@
 module SimpleNonlinearSolveTrackerExt
 
-using DiffEqBase, SciMLBase, SimpleNonlinearSolve, Tracker
+using DiffEqBase: DiffEqBase
+using SciMLBase: TrackerOriginator, NonlinearProblem, NonlinearLeastSquaresProblem
+using SimpleNonlinearSolve: SimpleNonlinearSolve
+using Tracker: Tracker, TrackedArray
 
-function SimpleNonlinearSolve.__internal_solve_up(prob::NonlinearProblem,
+function SimpleNonlinearSolve.__internal_solve_up(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
         sensealg, u0::TrackedArray, u0_changed, p, p_changed, alg, args...; kwargs...)
     return Tracker.track(
         SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0, u0_changed,
@@ -10,21 +14,23 @@ function SimpleNonlinearSolve.__internal_solve_up(prob::NonlinearProblem,
 end
 
 function SimpleNonlinearSolve.__internal_solve_up(
-        prob::NonlinearProblem, sensealg, u0::TrackedArray, u0_changed,
-        p::TrackedArray, p_changed, alg, args...; kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
+        u0::TrackedArray, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
     return Tracker.track(
         SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0, u0_changed,
         p, p_changed, alg, args...; kwargs...)
 end
 
-function SimpleNonlinearSolve.__internal_solve_up(prob::NonlinearProblem,
+function SimpleNonlinearSolve.__internal_solve_up(
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
         sensealg, u0, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
     return Tracker.track(
         SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0, u0_changed,
         p, p_changed, alg, args...; kwargs...)
 end
 
-Tracker.@grad function SimpleNonlinearSolve.__internal_solve_up(_prob::NonlinearProblem,
+Tracker.@grad function SimpleNonlinearSolve.__internal_solve_up(
+        _prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
         sensealg, u0_, u0_changed, p_, p_changed, alg, args...; kwargs...)
     u0, p = Tracker.data(u0_), Tracker.data(p_)
     prob = remake(_prob; u0, p)
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
index b29a1529a..559930b08 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveZygoteExt.jl
@@ -1,6 +1,7 @@
 module SimpleNonlinearSolveZygoteExt
 
-import SimpleNonlinearSolve, Zygote
+using SimpleNonlinearSolve: SimpleNonlinearSolve
+using Zygote: Zygote
 
 SimpleNonlinearSolve.__is_extension_loaded(::Val{:Zygote}) = true
 
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 3a4e5ebdb..e38aaa54b 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -8,14 +8,12 @@ import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidat
 
     import DiffEqBase: AbstractNonlinearTerminationMode,
                        AbstractSafeNonlinearTerminationMode,
-                       AbstractSafeBestNonlinearTerminationMode,
-                       NonlinearSafeTerminationReturnCode, get_termination_mode,
-                       NONLINEARSOLVE_DEFAULT_NORM
+                       AbstractSafeBestNonlinearTerminationMode, NONLINEARSOLVE_DEFAULT_NORM
     import DiffResults
     import ForwardDiff: Dual
     import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
     import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace, _unwrap_val
-    import StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, MMatrix, Size
+    import StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
 end
 
 @reexport using ADTypes, SciMLBase
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 76e91fcb4..43689a29d 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -77,7 +77,7 @@ except `cache` (& `J` if not nothing) are mutated.
 function value_and_jacobian(ad, f::F, y, x::X, p, cache; J = nothing) where {F, X}
     if isinplace(f)
         _f = (du, u) -> f(du, u, p)
-        if DiffEqBase.has_jac(f)
+        if SciMLBase.has_jac(f)
             f.jac(J, x, p)
             _f(y, x)
             return y, J
@@ -97,7 +97,7 @@ function value_and_jacobian(ad, f::F, y, x::X, p, cache; J = nothing) where {F,
         end
     else
         _f = Base.Fix2(f, p)
-        if DiffEqBase.has_jac(f)
+        if SciMLBase.has_jac(f)
             return _f(x), f.jac(x, p)
         elseif ad isa AutoForwardDiff
             if ArrayInterface.can_setindex(x)
@@ -124,7 +124,7 @@ end
 function __polyester_forwarddiff_jacobian! end
 
 function value_and_jacobian(ad, f::F, y, x::Number, p, cache; J = nothing) where {F}
-    if DiffEqBase.has_jac(f)
+    if SciMLBase.has_jac(f)
         return f(x, p), f.jac(x, p)
     elseif ad isa AutoForwardDiff
         T = typeof(__standard_tag(ad.tag, x))
@@ -152,7 +152,7 @@ function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
     if isinplace(f)
         _f = (du, u) -> f(du, u, p)
         J = similar(y, length(y), length(x))
-        if DiffEqBase.has_jac(f)
+        if SciMLBase.has_jac(f)
             return J, nothing
         elseif ad isa AutoForwardDiff || ad isa AutoPolyesterForwardDiff
             return J, __get_jacobian_config(ad, _f, y, x)
@@ -163,7 +163,7 @@ function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
         end
     else
         _f = Base.Fix2(f, p)
-        if DiffEqBase.has_jac(f)
+        if SciMLBase.has_jac(f)
             return nothing, nothing
         elseif ad isa AutoForwardDiff
             J = ArrayInterface.can_setindex(x) ? similar(y, length(y), length(x)) : nothing

From c5d1a7d7a190de274eebf4c42e2fdbe2cabebdad Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 14:27:43 -0700
Subject: [PATCH 334/375] Run formatter

---
 lib/SimpleNonlinearSolve/.JuliaFormatter.toml |  3 +-
 .../SimpleNonlinearSolveChainRulesCoreExt.jl  | 12 +--
 ...leNonlinearSolvePolyesterForwardDiffExt.jl |  8 +-
 .../ext/SimpleNonlinearSolveReverseDiffExt.jl | 43 +++++------
 .../ext/SimpleNonlinearSolveTrackerExt.jl     | 27 +++----
 .../src/SimpleNonlinearSolve.jl               | 20 +++--
 lib/SimpleNonlinearSolve/src/ad.jl            | 34 +++++----
 .../src/bracketing/alefeld.jl                 | 49 +++++-------
 .../src/bracketing/bisection.jl               | 31 ++++----
 .../src/bracketing/brent.jl                   | 26 +++----
 .../src/bracketing/falsi.jl                   | 23 +++---
 .../src/bracketing/itp.jl                     | 23 +++---
 .../src/bracketing/ridder.jl                  | 27 ++++---
 lib/SimpleNonlinearSolve/src/linesearch.jl    | 14 ++--
 .../src/nlsolve/broyden.jl                    | 12 +--
 .../src/nlsolve/dfsane.jl                     |  8 +-
 .../src/nlsolve/halley.jl                     | 12 +--
 .../src/nlsolve/klement.jl                    |  8 +-
 .../src/nlsolve/lbroyden.jl                   | 74 +++++++++----------
 .../src/nlsolve/raphson.jl                    |  4 +-
 .../src/nlsolve/trustRegion.jl                | 12 +--
 lib/SimpleNonlinearSolve/src/utils.jl         | 38 +++++-----
 .../test/core/23_test_problems_tests.jl       | 11 ++-
 .../test/core/forward_ad_tests.jl             | 21 +++---
 .../test/core/least_squares_tests.jl          |  3 +-
 .../test/core/matrix_resizing_tests.jl        |  8 +-
 .../test/core/rootfind_tests.jl               | 60 +++++++--------
 .../test/gpu/cuda_tests.jl                    | 14 ++--
 28 files changed, 310 insertions(+), 315 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.JuliaFormatter.toml b/lib/SimpleNonlinearSolve/.JuliaFormatter.toml
index 4d06911d7..66c13bae3 100644
--- a/lib/SimpleNonlinearSolve/.JuliaFormatter.toml
+++ b/lib/SimpleNonlinearSolve/.JuliaFormatter.toml
@@ -1,4 +1,5 @@
 style = "sciml"
 format_markdown = true
 annotate_untyped_fields_with_any = false
-format_docstrings = true
\ No newline at end of file
+format_docstrings = true
+join_lines_based_on_source = false
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
index 814af8f39..2987f1c5b 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
@@ -8,14 +8,14 @@ using SimpleNonlinearSolve: SimpleNonlinearSolve
 # The expectation here is that no-one is using this directly inside a GPU kernel. We can
 # eventually lift this requirement using a custom adjoint
 function ChainRulesCore.rrule(::typeof(SimpleNonlinearSolve.__internal_solve_up),
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0,
-        u0_changed, p, p_changed, alg, args...; kwargs...)
-    out, ∇internal = DiffEqBase._solve_adjoint(prob, sensealg, u0, p,
-        ChainRulesOriginator(), alg, args...; kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+        sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+    out, ∇internal = DiffEqBase._solve_adjoint(
+        prob, sensealg, u0, p, ChainRulesOriginator(), alg, args...; kwargs...)
     function ∇__internal_solve_up(Δ)
         ∂f, ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
-        return (∂f, ∂prob, ∂sensealg, ∂u0, NoTangent(), ∂p, NoTangent(), NoTangent(),
-            ∂args...)
+        return (
+            ∂f, ∂prob, ∂sensealg, ∂u0, NoTangent(), ∂p, NoTangent(), NoTangent(), ∂args...)
     end
     return out, ∇__internal_solve_up
 end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
index aa38d1c4e..ac898ac16 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
@@ -5,14 +5,14 @@ using SimpleNonlinearSolve: SimpleNonlinearSolve
 
 @inline SimpleNonlinearSolve.__is_extension_loaded(::Val{:PolyesterForwardDiff}) = true
 
-@inline function SimpleNonlinearSolve.__polyester_forwarddiff_jacobian!(f!::F, y, J, x,
-        chunksize) where {F}
+@inline function SimpleNonlinearSolve.__polyester_forwarddiff_jacobian!(
+        f!::F, y, J, x, chunksize) where {F}
     PolyesterForwardDiff.threaded_jacobian!(f!, y, J, x, chunksize)
     return J
 end
 
-@inline function SimpleNonlinearSolve.__polyester_forwarddiff_jacobian!(f::F, J, x,
-        chunksize) where {F}
+@inline function SimpleNonlinearSolve.__polyester_forwarddiff_jacobian!(
+        f::F, J, x, chunksize) where {F}
     PolyesterForwardDiff.threaded_jacobian!(f, J, x, chunksize)
     return J
 end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
index f9bfe0965..a6a1c2dbf 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
@@ -9,52 +9,53 @@ using SimpleNonlinearSolve: SimpleNonlinearSolve
 function SimpleNonlinearSolve.__internal_solve_up(
         prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
         u0::TrackedArray, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0,
-        u0_changed, p, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
+        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
 function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0, u0_changed,
-        p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0,
-        u0_changed, p, p_changed, alg, args...; kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
+        u0, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
+        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
 function SimpleNonlinearSolve.__internal_solve_up(
         prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
         u0::TrackedArray, u0_changed, p, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0,
-        u0_changed, p, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
+        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
 function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
-        u0::AbstractArray{<:TrackedReal}, u0_changed, p::AbstractArray{<:TrackedReal},
-        p_changed, alg, args...; kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+        sensealg, u0::AbstractArray{<:TrackedReal}, u0_changed,
+        p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
     return SimpleNonlinearSolve.__internal_solve_up(
         prob, sensealg, ArrayInterface.aos_to_soa(u0), true,
         ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
 end
 
 function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0, u0_changed,
-        p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0,
+        u0_changed, p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
     return SimpleNonlinearSolve.__internal_solve_up(
-        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p), true, alg, args...;
-        kwargs...)
+        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
+        true, alg, args...; kwargs...)
 end
 
 function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
-        u0::AbstractArray{<:TrackedReal}, u0_changed, p, p_changed, alg, args...; kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+        sensealg, u0::AbstractArray{<:TrackedReal},
+        u0_changed, p, p_changed, alg, args...; kwargs...)
     return SimpleNonlinearSolve.__internal_solve_up(
-        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p), true, alg, args...;
-        kwargs...)
+        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
+        true, alg, args...; kwargs...)
 end
 
 ReverseDiff.@grad function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0,
-        u0_changed, p, p_changed, alg, args...; kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+        sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...)
     out, ∇internal = DiffEqBase._solve_adjoint(
         prob, sensealg, ReverseDiff.value(u0), ReverseDiff.value(p),
         ReverseDiffOriginator(), alg, args...; kwargs...)
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
index 7b35de09b..b49bd78cc 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
@@ -6,27 +6,24 @@ using SimpleNonlinearSolve: SimpleNonlinearSolve
 using Tracker: Tracker, TrackedArray
 
 function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
-        sensealg, u0::TrackedArray, u0_changed, p, p_changed, alg, args...; kwargs...)
-    return Tracker.track(
-        SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0, u0_changed,
-        p, p_changed, alg, args...; kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
+        u0::TrackedArray, u0_changed, p, p_changed, alg, args...; kwargs...)
+    return Tracker.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
+        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
 function SimpleNonlinearSolve.__internal_solve_up(
         prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
         u0::TrackedArray, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return Tracker.track(
-        SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0, u0_changed,
-        p, p_changed, alg, args...; kwargs...)
+    return Tracker.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
+        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
 function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
-        sensealg, u0, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return Tracker.track(
-        SimpleNonlinearSolve.__internal_solve_up, prob, sensealg, u0, u0_changed,
-        p, p_changed, alg, args...; kwargs...)
+        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
+        u0, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
+    return Tracker.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
+        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
 Tracker.@grad function SimpleNonlinearSolve.__internal_solve_up(
@@ -34,8 +31,8 @@ Tracker.@grad function SimpleNonlinearSolve.__internal_solve_up(
         sensealg, u0_, u0_changed, p_, p_changed, alg, args...; kwargs...)
     u0, p = Tracker.data(u0_), Tracker.data(p_)
     prob = remake(_prob; u0, p)
-    out, ∇internal = DiffEqBase._solve_adjoint(prob, sensealg, u0, p,
-        SciMLBase.TrackerOriginator(), alg, args...; kwargs...)
+    out, ∇internal = DiffEqBase._solve_adjoint(
+        prob, sensealg, u0, p, SciMLBase.TrackerOriginator(), alg, args...; kwargs...)
 
     function ∇__internal_solve_up(Δ)
         ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index e38aaa54b..dfd650c4d 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -56,17 +56,15 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Nothing, args...;
 end
 
 # By Pass the highlevel checks for NonlinearProblem for Simple Algorithms
-function SciMLBase.solve(
-        prob::NonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
+function SciMLBase.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
         args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...)
     if sensealg === nothing && haskey(prob.kwargs, :sensealg)
         sensealg = prob.kwargs[:sensealg]
     end
     new_u0 = u0 !== nothing ? u0 : prob.u0
     new_p = p !== nothing ? p : prob.p
-    return __internal_solve_up(
-        prob, sensealg, new_u0, u0 === nothing, new_p, p === nothing,
-        alg, args...; prob.kwargs..., kwargs...)
+    return __internal_solve_up(prob, sensealg, new_u0, u0 === nothing, new_p,
+        p === nothing, alg, args...; prob.kwargs..., kwargs...)
 end
 
 function __internal_solve_up(_prob::NonlinearProblem, sensealg, u0, u0_changed,
@@ -78,10 +76,10 @@ end
 @setup_workload begin
     for T in (Float32, Float64)
         prob_no_brack_scalar = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
-        prob_no_brack_iip = NonlinearProblem{true}((du, u, p) -> du .= u .* u .- p,
-            T.([1.0, 1.0, 1.0]), T(2))
-        prob_no_brack_oop = NonlinearProblem{false}((u, p) -> u .* u .- p,
-            T.([1.0, 1.0, 1.0]), T(2))
+        prob_no_brack_iip = NonlinearProblem{true}(
+            (du, u, p) -> du .= u .* u .- p, T.([1.0, 1.0, 1.0]), T(2))
+        prob_no_brack_oop = NonlinearProblem{false}(
+            (u, p) -> u .* u .- p, T.([1.0, 1.0, 1.0]), T(2))
 
         algs = [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
             SimpleTrustRegion(), SimpleLimitedMemoryBroyden(; threshold = 2)]
@@ -101,8 +99,8 @@ end
             end
         end
 
-        prob_brack = IntervalNonlinearProblem{false}((u, p) -> u * u - p,
-            T.((0.0, 2.0)), T(2))
+        prob_brack = IntervalNonlinearProblem{false}(
+            (u, p) -> u * u - p, T.((0.0, 2.0)), T(2))
         algs = [Bisection(), Falsi(), Ridder(), Brent(), Alefeld(), ITP()]
         @compile_workload begin
             for alg in algs
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index d4e091c43..f42651bfe 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -1,7 +1,9 @@
 function SciMLBase.solve(
-        prob::NonlinearProblem{<:Union{Number, <:AbstractArray},
-            iip, <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
-        alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) where {T, V, P, iip}
+        prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip,
+            <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...;
+        kwargs...) where {T, V, P, iip}
     sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
     dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
     return SciMLBase.build_solution(
@@ -9,9 +11,11 @@ function SciMLBase.solve(
 end
 
 function SciMLBase.solve(
-        prob::NonlinearLeastSquaresProblem{<:AbstractArray,
-            iip, <:Union{<:AbstractArray{<:Dual{T, V, P}}}},
-        alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) where {T, V, P, iip}
+        prob::NonlinearLeastSquaresProblem{
+            <:AbstractArray, iip, <:Union{<:AbstractArray{<:Dual{T, V, P}}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...;
+        kwargs...) where {T, V, P, iip}
     sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
     dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
     return SciMLBase.build_solution(
@@ -21,13 +25,16 @@ end
 for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
     @eval begin
         function SciMLBase.solve(
-                prob::IntervalNonlinearProblem{uType, iip,
-                    <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
-                alg::$(algType), args...; kwargs...) where {uType, T, V, P, iip}
+                prob::IntervalNonlinearProblem{
+                    uType, iip, <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
+                alg::$(algType),
+                args...;
+                kwargs...) where {uType, T, V, P, iip}
             sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
             dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
-            return SciMLBase.build_solution(prob, alg, dual_soln, sol.resid; sol.retcode,
-                sol.stats, sol.original, left = Dual{T, V, P}(sol.left, partials),
+            return SciMLBase.build_solution(
+                prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats,
+                sol.original, left = Dual{T, V, P}(sol.left, partials),
                 right = Dual{T, V, P}(sol.right, partials))
         end
     end
@@ -125,9 +132,8 @@ function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs..
             else
                 _F = @closure (u, p) -> begin
                     T = promote_type(eltype(u), eltype(p))
-                    res = DiffResults.DiffResult(
-                        similar(u, T, size(sol.resid)), similar(
-                            u, T, length(sol.resid), length(u)))
+                    res = DiffResults.DiffResult(similar(u, T, size(sol.resid)),
+                        similar(u, T, length(sol.resid), length(u)))
                     ForwardDiff.jacobian!(res, Base.Fix2(prob.f, p), u)
                     return reshape(
                         2 .* vec(DiffResults.value(res))' * DiffResults.jacobian(res),
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl b/lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl
index 3b89751a7..55020c8a6 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/alefeld.jl
@@ -15,12 +15,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
     c = a - (b - a) / (f(b) - f(a)) * f(a)
 
     fc = f(c)
-    (a == c || b == c) &&
-        return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
-            left = a, right = b)
-    iszero(fc) &&
-        return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success, left = a,
-            right = b)
+    (a == c || b == c) && return build_solution(
+        prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit, left = a, right = b)
+    iszero(fc) && return build_solution(
+        prob, alg, c, fc; retcode = ReturnCode.Success, left = a, right = b)
     a, b, d = _bracket(f, a, b, c)
     e = zero(a)   # Set e as 0 before iteration to avoid a non-value f(e)
 
@@ -38,12 +36,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
         end
         ē, fc = d, f(c)
         (a == c || b == c) &&
-            return build_solution(
-                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
-                left = a, right = b)
-        iszero(fc) &&
-            return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
+            return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
                 left = a, right = b)
+        iszero(fc) && return build_solution(
+            prob, alg, c, fc; retcode = ReturnCode.Success, left = a, right = b)
         ā, b̄, d̄ = _bracket(f, a, b, c)
 
         # The second bracketing block
@@ -58,12 +54,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
         end
         fc = f(c)
         (ā == c || b̄ == c) &&
-            return build_solution(
-                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
-                left = ā, right = b̄)
-        iszero(fc) &&
-            return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
+            return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
                 left = ā, right = b̄)
+        iszero(fc) && return build_solution(
+            prob, alg, c, fc; retcode = ReturnCode.Success, left = ā, right = b̄)
         ā, b̄, d̄ = _bracket(f, ā, b̄, c)
 
         # The third bracketing block
@@ -78,12 +72,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
         end
         fc = f(c)
         (ā == c || b̄ == c) &&
-            return build_solution(
-                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
-                left = ā, right = b̄)
-        iszero(fc) &&
-            return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
+            return build_solution(prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
                 left = ā, right = b̄)
+        iszero(fc) && return build_solution(
+            prob, alg, c, fc; retcode = ReturnCode.Success, left = ā, right = b̄)
         ā, b̄, d = _bracket(f, ā, b̄, c)
 
         # The last bracketing block
@@ -93,12 +85,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
             e = d
             c = 0.5 * (ā + b̄)
             fc = f(c)
-            (ā == c || b̄ == c) &&
-                return build_solution(prob, alg, c, fc;
-                    retcode = ReturnCode.FloatingPointLimit, left = ā, right = b̄)
-            iszero(fc) &&
-                return build_solution(prob, alg, c, fc; retcode = ReturnCode.Success,
-                    left = ā, right = b̄)
+            (ā == c || b̄ == c) && return build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+                left = ā, right = b̄)
+            iszero(fc) && return build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.Success, left = ā, right = b̄)
             a, b, d = _bracket(f, ā, b̄, c)
         end
     end
@@ -112,8 +103,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
     fc = f(c)
 
     # Reuturn solution when run out of max interation
-    return build_solution(prob, alg, c, fc; retcode = ReturnCode.MaxIters,
-        left = a, right = b)
+    return build_solution(
+        prob, alg, c, fc; retcode = ReturnCode.MaxIters, left = a, right = b)
 end
 
 # Define subrotine function bracket, check fc before bracket to return solution
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
index acadf6aa1..d55a0ce5e 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
@@ -19,25 +19,24 @@ A common bisection method.
     exact_right::Bool = false
 end
 
-function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...;
-        maxiters = 1000, abstol = nothing, kwargs...)
+function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection,
+        args...; maxiters = 1000, abstol = nothing, kwargs...)
     @assert !isinplace(prob) "`Bisection` only supports OOP problems."
     f = Base.Fix2(prob.f, prob.p)
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
-    abstol = __get_tolerance(nothing, abstol,
-        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+    abstol = __get_tolerance(
+        nothing, abstol, promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
-        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
-            left, right)
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
     end
 
     if iszero(fr)
         return build_solution(
-            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
-            left, right)
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
 
     i = 1
@@ -49,8 +48,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
                     retcode = ReturnCode.FloatingPointLimit)
             fm = f(mid)
             if abs((right - left) / 2) < abstol
-                return build_solution(prob, alg, mid, fm; retcode = ReturnCode.Success,
-                    left, right)
+                return build_solution(
+                    prob, alg, mid, fm; retcode = ReturnCode.Success, left, right)
             end
             if iszero(fm)
                 right = mid
@@ -67,8 +66,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection, args...
         end
     end
 
-    sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
-        maxiters = maxiters - i, prob, alg)
+    sol, i, left, right, fl, fr = __bisection(
+        left, right, fl, fr, f; abstol, maxiters = maxiters - i, prob, alg)
 
     sol !== nothing && return sol
 
@@ -81,15 +80,15 @@ function __bisection(left, right, fl, fr, f::F; abstol, maxiters, prob, alg) whe
     while i < maxiters
         mid = (left + right) / 2
         if (mid == left || mid == right)
-            sol = build_solution(prob, alg, left, fl; left, right,
-                retcode = ReturnCode.FloatingPointLimit)
+            sol = build_solution(
+                prob, alg, left, fl; left, right, retcode = ReturnCode.FloatingPointLimit)
             break
         end
 
         fm = f(mid)
         if abs((right - left) / 2) < abstol
-            sol = build_solution(prob, alg, mid, fm; left, right,
-                retcode = ReturnCode.Success)
+            sol = build_solution(
+                prob, alg, mid, fm; left, right, retcode = ReturnCode.Success)
             break
         end
 
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
index 89b2e60be..649286e03 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
@@ -13,18 +13,17 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
     fl, fr = f(left), f(right)
     ϵ = eps(convert(typeof(fl), 1))
 
-    abstol = __get_tolerance(nothing, abstol,
-        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+    abstol = __get_tolerance(
+        nothing, abstol, promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
-        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
-            left, right)
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
     end
 
     if iszero(fr)
         return build_solution(
-            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
-            left, right)
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
 
     if abs(fl) < abs(fr)
@@ -60,18 +59,17 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
                (!cond && abs(c - d) ≤ ϵ)
                 # Bisection method
                 s = (left + right) / 2
-                (s == left || s == right) &&
-                    return SciMLBase.build_solution(prob, alg, left, fl;
-                        retcode = ReturnCode.FloatingPointLimit,
-                        left = left, right = right)
+                (s == left || s == right) && return SciMLBase.build_solution(
+                    prob, alg, left, fl; retcode = ReturnCode.FloatingPointLimit,
+                    left = left, right = right)
                 cond = true
             else
                 cond = false
             end
             fs = f(s)
             if abs((right - left) / 2) < abstol
-                return SciMLBase.build_solution(prob, alg, s, fs;
-                    retcode = ReturnCode.Success,
+                return SciMLBase.build_solution(
+                    prob, alg, s, fs; retcode = ReturnCode.Success,
                     left = left, right = right)
             end
             if iszero(fs)
@@ -105,8 +103,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
         end
     end
 
-    sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
-        maxiters = maxiters - i, prob, alg)
+    sol, i, left, right, fl, fr = __bisection(
+        left, right, fl, fr, f; abstol, maxiters = maxiters - i, prob, alg)
 
     sol !== nothing && return sol
 
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
index 896e07329..ee78b73fc 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
@@ -12,18 +12,17 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
-    abstol = __get_tolerance(nothing, abstol,
-        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+    abstol = __get_tolerance(
+        nothing, abstol, promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
-        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
-            left, right)
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
     end
 
     if iszero(fr)
         return build_solution(
-            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
-            left, right)
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
 
     # Regula Falsi Steps
@@ -44,8 +43,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
 
             fm = f(mid)
             if abs((right - left) / 2) < abstol
-                return build_solution(prob, alg, mid, fm; left, right,
-                    retcode = ReturnCode.Success)
+                return build_solution(
+                    prob, alg, mid, fm; left, right, retcode = ReturnCode.Success)
             end
 
             if abs(fm) < abstol
@@ -62,10 +61,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
         end
     end
 
-    sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
-        maxiters = maxiters - i, prob, alg)
+    sol, i, left, right, fl, fr = __bisection(
+        left, right, fl, fr, f; abstol, maxiters = maxiters - i, prob, alg)
     sol !== nothing && return sol
 
-    return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
-        left, right)
+    return SciMLBase.build_solution(
+        prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
 end
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index 2926dfd7a..2972d1c5c 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -58,18 +58,17 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
-    abstol = __get_tolerance(nothing, abstol,
-        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+    abstol = __get_tolerance(
+        nothing, abstol, promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
-        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
-            left, right)
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
     end
 
     if iszero(fr)
         return build_solution(
-            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
-            left, right)
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
     ϵ = abstol
     #defining variables/cache
@@ -110,8 +109,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
         end
 
         if abs((left - right) / 2) < ϵ
-            return build_solution(prob, alg, mid, f(mid); retcode = ReturnCode.Success,
-                left, right)
+            return build_solution(
+                prob, alg, mid, f(mid); retcode = ReturnCode.Success, left, right)
         end
 
         ## Update ##
@@ -127,16 +126,16 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
             left = xp
             fl = yp
         else
-            return build_solution(prob, alg, xp, yps; retcode = ReturnCode.Success,
-                left = xp, right = xp)
+            return build_solution(
+                prob, alg, xp, yps; retcode = ReturnCode.Success, left = xp, right = xp)
         end
         i += 1
         mid = (left + right) / 2
         ϵ_s /= 2
 
         if __nextfloat_tdir(left, prob.tspan...) == right
-            return build_solution(prob, alg, left, fl; left, right,
-                retcode = ReturnCode.FloatingPointLimit)
+            return build_solution(
+                prob, alg, left, fl; left, right, retcode = ReturnCode.FloatingPointLimit)
         end
     end
 
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
index 3b23f4287..a974824c2 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
@@ -12,18 +12,17 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
     left, right = prob.tspan
     fl, fr = f(left), f(right)
 
-    abstol = __get_tolerance(nothing, abstol,
-        promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
+    abstol = __get_tolerance(
+        nothing, abstol, promote_type(eltype(first(prob.tspan)), eltype(last(prob.tspan))))
 
     if iszero(fl)
-        return build_solution(prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft,
-            left, right)
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
     end
 
     if iszero(fr)
         return build_solution(
-            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight,
-            left, right)
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
 
     xo = oftype(left, Inf)
@@ -37,15 +36,15 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
             fm = f(mid)
             s = sqrt(fm^2 - fl * fr)
             if iszero(s)
-                return build_solution(prob, alg, left, fl; left, right,
-                    retcode = ReturnCode.Failure)
+                return build_solution(
+                    prob, alg, left, fl; left, right, retcode = ReturnCode.Failure)
             end
             x = mid + (mid - left) * sign(fl - fr) * fm / s
             fx = f(x)
             xo = x
             if abs((right - left) / 2) < abstol
-                return build_solution(prob, alg, mid, fm; retcode = ReturnCode.Success,
-                    left, right)
+                return build_solution(
+                    prob, alg, mid, fm; retcode = ReturnCode.Success, left, right)
             end
             if iszero(fx)
                 right = x
@@ -69,10 +68,10 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
         end
     end
 
-    sol, i, left, right, fl, fr = __bisection(left, right, fl, fr, f; abstol,
-        maxiters = maxiters - i, prob, alg)
+    sol, i, left, right, fl, fr = __bisection(
+        left, right, fl, fr, f; abstol, maxiters = maxiters - i, prob, alg)
     sol !== nothing && return sol
 
-    return SciMLBase.build_solution(prob, alg, left, fl; retcode = ReturnCode.MaxIters,
-        left, right)
+    return SciMLBase.build_solution(
+        prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
 end
diff --git a/lib/SimpleNonlinearSolve/src/linesearch.jl b/lib/SimpleNonlinearSolve/src/linesearch.jl
index c33253f63..b8e830af2 100644
--- a/lib/SimpleNonlinearSolve/src/linesearch.jl
+++ b/lib/SimpleNonlinearSolve/src/linesearch.jl
@@ -37,8 +37,8 @@ end
 end
 
 (alg::LiFukushimaLineSearch)(prob, fu, u) = __generic_init(alg, prob, fu, u)
-function (alg::LiFukushimaLineSearch)(prob, fu::Union{Number, SArray},
-        u::Union{Number, SArray})
+function (alg::LiFukushimaLineSearch)(
+        prob, fu::Union{Number, SArray}, u::Union{Number, SArray})
     (alg.nan_maxiters === missing || alg.nan_maxiters === nothing) &&
         return __static_init(alg, prob, fu, u)
     @warn "`LiFukushimaLineSearch` with NaN checking is not non-allocating" maxlog=1
@@ -57,14 +57,16 @@ function __generic_init(alg::LiFukushimaLineSearch, prob, fu, u)
 
     nan_maxiters = ifelse(alg.nan_maxiters === missing, 5, alg.nan_maxiters)
 
-    return LiFukushimaLineSearchCache(ϕ, T(alg.lambda_0), T(alg.beta), T(alg.sigma_1),
-        T(alg.sigma_2), T(alg.eta), T(alg.rho), T(true), nan_maxiters, alg.maxiters)
+    return LiFukushimaLineSearchCache(
+        ϕ, T(alg.lambda_0), T(alg.beta), T(alg.sigma_1), T(alg.sigma_2),
+        T(alg.eta), T(alg.rho), T(true), nan_maxiters, alg.maxiters)
 end
 
 function __static_init(alg::LiFukushimaLineSearch, prob, fu, u)
     T = promote_type(eltype(fu), eltype(u))
-    return StaticLiFukushimaLineSearchCache(prob.f, prob.p, T(alg.lambda_0), T(alg.beta),
-        T(alg.sigma_1), T(alg.sigma_2), T(alg.eta), T(alg.rho), alg.maxiters)
+    return StaticLiFukushimaLineSearchCache(
+        prob.f, prob.p, T(alg.lambda_0), T(alg.beta), T(alg.sigma_1),
+        T(alg.sigma_2), T(alg.eta), T(alg.rho), alg.maxiters)
 end
 
 function (cache::LiFukushimaLineSearchCache)(u, δu)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index 6fe121481..890578f5d 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -23,8 +23,8 @@ end
 __get_linesearch(::SimpleBroyden{LS}) where {LS} = Val(LS)
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
-        termination_condition = nothing, kwargs...)
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
     fx = _get_fx(prob, x)
     T = promote_type(eltype(x), eltype(fx))
@@ -48,11 +48,11 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
     @bb δJ⁻¹n = copy(x)
     @bb δJ⁻¹ = copy(J⁻¹)
 
-    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
-        termination_condition)
+    abstol, reltol, tc_cache = init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition)
 
-    ls_cache = __get_linesearch(alg) === Val(true) ?
-               LiFukushimaLineSearch()(prob, fx, x) : nothing
+    ls_cache = __get_linesearch(alg) === Val(true) ? LiFukushimaLineSearch()(prob, fx, x) :
+               nothing
 
     for _ in 1:maxiters
         @bb δx = J⁻¹ × vec(fprev)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 9f092648d..7dd152277 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -50,8 +50,8 @@ end
 function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
         M::Union{Int, Val} = Val(10), γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
         nexp::Int = 2, η_strategy::F = (f_1, k, x, F) -> f_1 ./ k^2) where {F}
-    return SimpleDFSane{_unwrap_val(M)}(σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp,
-        η_strategy)
+    return SimpleDFSane{_unwrap_val(M)}(
+        σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp, η_strategy)
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...;
@@ -70,8 +70,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...
     τ_min = T(alg.τ_min)
     τ_max = T(alg.τ_max)
 
-    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
-        termination_condition)
+    abstol, reltol, tc_cache = init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition)
 
     fx_norm = NONLINEARSOLVE_DEFAULT_NORM(fx)^nexp
     α_1 = one(T)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 934dc4763..e550258ef 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -24,8 +24,8 @@ A low-overhead implementation of Halley's Method.
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
-        termination_condition = nothing, kwargs...)
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
     isinplace(prob) &&
         error("SimpleHalley currently only supports out-of-place nonlinear problems")
 
@@ -34,8 +34,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
     T = eltype(x)
 
     autodiff = __get_concrete_autodiff(prob, alg.autodiff)
-    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
-        termination_condition)
+    abstol, reltol, tc_cache = init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition)
 
     @bb xo = copy(x)
 
@@ -59,8 +59,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
             dfx
         else
             fact = lu(dfx; check = false)
-            !issuccess(fact) && return build_solution(prob, alg, x, fx;
-                retcode = ReturnCode.Unstable)
+            !issuccess(fact) &&
+                return build_solution(prob, alg, x, fx; retcode = ReturnCode.Unstable)
             fact
         end
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index c2c8b446f..8041ef4b8 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -7,14 +7,14 @@ method is non-allocating on scalar and static array problems.
 struct SimpleKlement <: AbstractSimpleNonlinearSolveAlgorithm end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
-        termination_condition = nothing, kwargs...)
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
     T = eltype(x)
     fx = _get_fx(prob, x)
 
-    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
-        termination_condition)
+    abstol, reltol, tc_cache = init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition)
 
     @bb δx = copy(x)
     @bb fprev = copy(fx)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 600892edd..b34d4cddf 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -24,8 +24,8 @@ end
 __get_threshold(::SimpleLimitedMemoryBroyden{threshold}) where {threshold} = Val(threshold)
 __use_linesearch(::SimpleLimitedMemoryBroyden{Th, LS}) where {Th, LS} = Val(LS)
 
-function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27),
-        linesearch = Val(false), alpha = nothing)
+function SimpleLimitedMemoryBroyden(;
+        threshold::Union{Val, Int} = Val(27), linesearch = Val(false), alpha = nothing)
     return SimpleLimitedMemoryBroyden{_unwrap_val(threshold), _unwrap_val(linesearch)}(alpha)
 end
 
@@ -45,24 +45,25 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyd
 end
 
 @views function __generic_solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
-        args...; abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
-        termination_condition = nothing, kwargs...)
+        args...; abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
     threshold = __get_threshold(alg)
     η = min(_unwrap_val(threshold), maxiters)
 
     # For scalar problems / if the threshold is larger than problem size just use Broyden
     if x isa Number || length(x) ≤ η
-        return SciMLBase.__solve(prob, SimpleBroyden(; linesearch = __use_linesearch(alg)),
-            args...; abstol, reltol, maxiters, termination_condition, kwargs...)
+        return SciMLBase.__solve(
+            prob, SimpleBroyden(; linesearch = __use_linesearch(alg)), args...;
+            abstol, reltol, maxiters, termination_condition, kwargs...)
     end
 
     fx = _get_fx(prob, x)
 
     U, Vᵀ = __init_low_rank_jacobian(x, fx, x isa StaticArray ? threshold : Val(η))
 
-    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
-        termination_condition)
+    abstol, reltol, tc_cache = init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition)
 
     @bb xo = copy(x)
     @bb δx = copy(fx)
@@ -74,8 +75,8 @@ end
     Tcache = __lbroyden_threshold_cache(x, x isa StaticArray ? threshold : Val(η))
     @bb mat_cache = copy(x)
 
-    ls_cache = __use_linesearch(alg) === Val(true) ?
-               LiFukushimaLineSearch()(prob, fx, x) : nothing
+    ls_cache = __use_linesearch(alg) === Val(true) ? LiFukushimaLineSearch()(prob, fx, x) :
+               nothing
 
     for i in 1:maxiters
         α = ls_cache === nothing ? true : ls_cache(x, δx)
@@ -125,8 +126,8 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
 
     xo, δx, fo, δf = x, -fx, fx, fx
 
-    ls_cache = __use_linesearch(alg) === Val(true) ?
-               LiFukushimaLineSearch()(prob, fx, x) : nothing
+    ls_cache = __use_linesearch(alg) === Val(true) ? LiFukushimaLineSearch()(prob, fx, x) :
+               nothing
 
     T = promote_type(eltype(x), eltype(fx))
     if alg.alpha === nothing
@@ -138,8 +139,7 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
     end
 
     converged, res = __unrolled_lbroyden_initial_iterations(
-        prob, xo, fo, δx, abstol, U, Vᵀ,
-        threshold, ls_cache, init_α)
+        prob, xo, fo, δx, abstol, U, Vᵀ, threshold, ls_cache, init_α)
 
     converged &&
         return build_solution(prob, alg, res.x, res.fx; retcode = ReturnCode.Success)
@@ -173,39 +173,39 @@ function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemo
     return build_solution(prob, alg, xo, fo; retcode = ReturnCode.MaxIters)
 end
 
-@generated function __unrolled_lbroyden_initial_iterations(prob, xo, fo, δx, abstol, U,
-        Vᵀ, ::Val{threshold}, ls_cache, init_α) where {threshold}
+@generated function __unrolled_lbroyden_initial_iterations(
+        prob, xo, fo, δx, abstol, U, Vᵀ, ::Val{threshold},
+        ls_cache, init_α) where {threshold}
     calls = []
     for i in 1:threshold
         static_idx, static_idx_p1 = Val(i - 1), Val(i)
-        push!(calls,
-            quote
-                α = ls_cache === nothing ? true : ls_cache(xo, δx)
-                x = xo .+ α .* δx
-                fx = prob.f(x, prob.p)
-                δf = fx - fo
+        push!(calls, quote
+            α = ls_cache === nothing ? true : ls_cache(xo, δx)
+            x = xo .+ α .* δx
+            fx = prob.f(x, prob.p)
+            δf = fx - fo
 
-                maximum(abs, fx) ≤ abstol && return true, (; x, fx, δx)
+            maximum(abs, fx) ≤ abstol && return true, (; x, fx, δx)
 
-                _U = __first_n_getindex(U, $(static_idx))
-                _Vᵀ = __first_n_getindex(Vᵀ, $(static_idx))
+            _U = __first_n_getindex(U, $(static_idx))
+            _Vᵀ = __first_n_getindex(Vᵀ, $(static_idx))
 
-                vᵀ = _restructure(x, _rmatvec!!(_U, _Vᵀ, vec(δx), init_α))
-                mvec = _restructure(x, _matvec!!(_U, _Vᵀ, vec(δf), init_α))
+            vᵀ = _restructure(x, _rmatvec!!(_U, _Vᵀ, vec(δx), init_α))
+            mvec = _restructure(x, _matvec!!(_U, _Vᵀ, vec(δf), init_α))
 
-                d = dot(vᵀ, δf)
-                δx = @. (δx - mvec) / d
+            d = dot(vᵀ, δf)
+            δx = @. (δx - mvec) / d
 
-                U = Base.setindex(U, vec(δx), $(i))
-                Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), $(i))
+            U = Base.setindex(U, vec(δx), $(i))
+            Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), $(i))
 
-                _U = __first_n_getindex(U, $(static_idx_p1))
-                _Vᵀ = __first_n_getindex(Vᵀ, $(static_idx_p1))
-                δx = -_restructure(fx, _matvec!!(_U, _Vᵀ, vec(fx), init_α))
+            _U = __first_n_getindex(U, $(static_idx_p1))
+            _Vᵀ = __first_n_getindex(Vᵀ, $(static_idx_p1))
+            δx = -_restructure(fx, _matvec!!(_U, _Vᵀ, vec(fx), init_α))
 
-                xo = x
-                fo = fx
-            end)
+            xo = x
+            fo = fx
+        end)
     end
     push!(calls, quote
         # Termination Check
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
index 9735d0c8c..7b419ce99 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
@@ -32,8 +32,8 @@ function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresPr
     @bb xo = copy(x)
     J, jac_cache = jacobian_cache(autodiff, prob.f, fx, x, prob.p)
 
-    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
-        termination_condition)
+    abstol, reltol, tc_cache = init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition)
 
     for i in 1:maxiters
         fx, dfx = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index e6ccf6536..a19cf2c2d 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -56,8 +56,8 @@ scalar and static array problems.
 end
 
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
-        termination_condition = nothing, kwargs...)
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
     T = eltype(real(x))
     Δₘₐₓ = T(alg.max_trust_radius)
@@ -88,8 +88,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
     J, jac_cache = jacobian_cache(autodiff, prob.f, fx, x, prob.p)
     fx, ∇f = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
 
-    abstol, reltol, tc_cache = init_termination_cache(prob, abstol, reltol, fx, x,
-        termination_condition)
+    abstol, reltol, tc_cache = init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition)
 
     # Set default trust region radius if not specified by user.
     Δₘₐₓ == 0 && (Δₘₐₓ = max(norm_fx, maximum(x) - minimum(x)))
@@ -132,8 +132,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
         else
             Δ = t₁ * Δ
             shrink_counter += 1
-            shrink_counter > max_shrink_times && return build_solution(prob, alg, x, fx;
-                retcode = ReturnCode.ShrinkThresholdExceeded)
+            shrink_counter > max_shrink_times && return build_solution(
+                prob, alg, x, fx; retcode = ReturnCode.ShrinkThresholdExceeded)
         end
 
         if r ≥ η₁
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 43689a29d..333e54d3b 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -58,8 +58,8 @@ function __forwarddiff_jacobian_config(
         f::F, x::SArray, ck::ForwardDiff.Chunk{N}, tag) where {F, N}
     seeds = ForwardDiff.construct_seeds(ForwardDiff.Partials{N, eltype(x)})
     duals = ForwardDiff.Dual{typeof(tag), eltype(x), N}.(x)
-    return ForwardDiff.JacobianConfig{typeof(tag), eltype(x), N, typeof(duals)}(seeds,
-        duals)
+    return ForwardDiff.JacobianConfig{typeof(tag), eltype(x), N, typeof(duals)}(
+        seeds, duals)
 end
 
 function __get_jacobian_config(ad::AutoPolyesterForwardDiff{CS}, args...) where {CS}
@@ -205,8 +205,8 @@ end
 
 function compute_jacobian_and_hessian(ad::AutoFiniteDiff, prob, _, x::Number)
     fx = prob.f(x, prob.p)
-    J_fn = x -> FiniteDiff.finite_difference_derivative(Base.Fix2(prob.f, prob.p), x,
-        ad.fdtype)
+    J_fn = x -> FiniteDiff.finite_difference_derivative(
+        Base.Fix2(prob.f, prob.p), x, ad.fdtype)
     dfx = J_fn(x)
     d2fx = FiniteDiff.finite_difference_derivative(J_fn, x, ad.fdtype)
     return fx, dfx, d2fx
@@ -262,7 +262,8 @@ end
 @inline _restructure(y, x) = ArrayInterface.restructure(y, x)
 
 @inline function _get_fx(prob::NonlinearLeastSquaresProblem, x)
-    isinplace(prob) && prob.f.resid_prototype === nothing &&
+    isinplace(prob) &&
+        prob.f.resid_prototype === nothing &&
         error("Inplace NonlinearLeastSquaresProblem requires a `resid_prototype`")
     return _get_fx(prob.f, x, prob.p)
 end
@@ -289,17 +290,16 @@ end
 # is meant for low overhead solvers, users can opt into the other termination modes but the
 # default is to use the least overhead version.
 function init_termination_cache(prob::NonlinearProblem, abstol, reltol, du, u, ::Nothing)
-    return init_termination_cache(prob, abstol, reltol, du, u,
-        AbsNormTerminationMode(Base.Fix1(maximum, abs)))
+    return init_termination_cache(
+        prob, abstol, reltol, du, u, AbsNormTerminationMode(Base.Fix1(maximum, abs)))
 end
 function init_termination_cache(
         prob::NonlinearLeastSquaresProblem, abstol, reltol, du, u, ::Nothing)
-    return init_termination_cache(prob, abstol, reltol, du, u,
-        AbsNormTerminationMode(Base.Fix2(norm, 2)))
+    return init_termination_cache(
+        prob, abstol, reltol, du, u, AbsNormTerminationMode(Base.Fix2(norm, 2)))
 end
 
-function init_termination_cache(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+function init_termination_cache(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
         abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode)
     T = promote_type(eltype(du), eltype(u))
     abstol = __get_tolerance(u, abstol, T)
@@ -316,23 +316,23 @@ function init_termination_cache(
 end
 
 function check_termination(tc_cache, fx, x, xo, prob, alg)
-    return check_termination(tc_cache, fx, x, xo, prob, alg,
-        DiffEqBase.get_termination_mode(tc_cache))
+    return check_termination(
+        tc_cache, fx, x, xo, prob, alg, DiffEqBase.get_termination_mode(tc_cache))
 end
-function check_termination(tc_cache, fx, x, xo, prob, alg,
-        ::AbstractNonlinearTerminationMode)
+function check_termination(
+        tc_cache, fx, x, xo, prob, alg, ::AbstractNonlinearTerminationMode)
     tc_cache(fx, x, xo) &&
         return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
     return nothing
 end
-function check_termination(tc_cache, fx, x, xo, prob, alg,
-        ::AbstractSafeNonlinearTerminationMode)
+function check_termination(
+        tc_cache, fx, x, xo, prob, alg, ::AbstractSafeNonlinearTerminationMode)
     tc_cache(fx, x, xo) &&
         return build_solution(prob, alg, x, fx; retcode = tc_cache.retcode)
     return nothing
 end
-function check_termination(tc_cache, fx, x, xo, prob, alg,
-        ::AbstractSafeBestNonlinearTerminationMode)
+function check_termination(
+        tc_cache, fx, x, xo, prob, alg, ::AbstractSafeBestNonlinearTerminationMode)
     if tc_cache(fx, x, xo)
         if isinplace(prob)
             prob.f(fx, x, prob.p)
diff --git a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
index 2180943fc..9625c6872 100644
--- a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
@@ -4,8 +4,8 @@ using LinearAlgebra, NonlinearProblemLibrary, DiffEqBase, Test
 problems = NonlinearProblemLibrary.problems
 dicts = NonlinearProblemLibrary.dicts
 
-function test_on_library(problems, dicts, alg_ops, broken_tests, ϵ = 1e-4;
-        skip_tests = nothing)
+function test_on_library(
+        problems, dicts, alg_ops, broken_tests, ϵ = 1e-4; skip_tests = nothing)
     for (idx, (problem, dict)) in enumerate(zip(problems, dicts))
         x = dict["start"]
         res = similar(x)
@@ -13,8 +13,8 @@ function test_on_library(problems, dicts, alg_ops, broken_tests, ϵ = 1e-4;
         @testset "$idx: $(dict["title"])" begin
             for alg in alg_ops
                 try
-                    sol = solve(nlprob, alg;
-                        termination_condition = AbsNormTerminationMode())
+                    sol = solve(
+                        nlprob, alg; termination_condition = AbsNormTerminationMode())
                     problem(res, sol.u, nothing)
 
                     skip = skip_tests !== nothing && idx in skip_tests[alg]
@@ -51,8 +51,7 @@ end
 end
 
 @testitem "SimpleTrustRegion" setup=[RobustnessTesting] tags=[:core] begin
-    alg_ops = (SimpleTrustRegion(),
-        SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
+    alg_ops = (SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
     broken_tests[alg_ops[1]] = [3, 15, 16, 21]
diff --git a/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
index 50fc18e94..ab1db6cb0 100644
--- a/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/forward_ad_tests.jl
@@ -41,8 +41,9 @@ export test_f, test_f!, jacobian_f, solve_with, __compatible
 end
 
 @testitem "ForwardDiff.jl Integration: Rootfinding" setup=[ForwardADRootfindingTesting] tags=[:core] begin
-    @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
-        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleNewtonRaphson(), SimpleTrustRegion(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
         SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleDFSane())
         us = (2.0, @SVector[1.0, 1.0], [1.0, 1.0], ones(2, 2), @SArray ones(2, 2))
 
@@ -171,25 +172,27 @@ end
         function obj_4(p)
             prob_iip = NonlinearLeastSquaresProblem(
                 NonlinearFunction{true}(
-                    loss_function!; resid_prototype = zeros(length(y_target))), θ_init, p)
+                    loss_function!; resid_prototype = zeros(length(y_target))),
+                θ_init,
+                p)
             sol = solve(prob_iip, alg)
             return sum(abs2, sol.u)
         end
 
         function obj_5(p)
             ff = NonlinearFunction{true}(
-                loss_function!; resid_prototype = zeros(length(y_target)), jac = loss_function_jac!)
-            prob_iip = NonlinearLeastSquaresProblem(
-                ff, θ_init, p)
+                loss_function!; resid_prototype = zeros(length(y_target)),
+                jac = loss_function_jac!)
+            prob_iip = NonlinearLeastSquaresProblem(ff, θ_init, p)
             sol = solve(prob_iip, alg)
             return sum(abs2, sol.u)
         end
 
         function obj_6(p)
             ff = NonlinearFunction{true}(
-                loss_function!; resid_prototype = zeros(length(y_target)), vjp = loss_function_vjp!)
-            prob_iip = NonlinearLeastSquaresProblem(
-                ff, θ_init, p)
+                loss_function!; resid_prototype = zeros(length(y_target)),
+                vjp = loss_function_vjp!)
+            prob_iip = NonlinearLeastSquaresProblem(ff, θ_init, p)
             sol = solve(prob_iip, alg)
             return sum(abs2, sol.u)
         end
diff --git a/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl b/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
index ef3a05504..005c463ff 100644
--- a/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/least_squares_tests.jl
@@ -29,7 +29,8 @@
     end
 
     prob_iip = NonlinearLeastSquaresProblem(
-        NonlinearFunction{true}(loss_function!, resid_prototype = zeros(length(y_target))), θ_init, x)
+        NonlinearFunction{true}(loss_function!, resid_prototype = zeros(length(y_target))),
+        θ_init, x)
 
     @testset "Solver: $(nameof(typeof(solver)))" for solver in [
         SimpleNewtonRaphson(AutoForwardDiff()), SimpleGaussNewton(AutoForwardDiff()),
diff --git a/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
index 17cd3d674..3f7dfbb40 100644
--- a/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
@@ -5,10 +5,10 @@
     vecprob = NonlinearProblem(ff, vec(u0), p)
     prob = NonlinearProblem(ff, u0, p)
 
-    @testset "$(nameof(typeof(alg)))" for alg in (SimpleKlement(), SimpleBroyden(),
-        SimpleNewtonRaphson(), SimpleDFSane(),
-        SimpleLimitedMemoryBroyden(; threshold = Val(2)),
-        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleKlement(), SimpleBroyden(), SimpleNewtonRaphson(), SimpleDFSane(),
+        SimpleLimitedMemoryBroyden(; threshold = Val(2)), SimpleTrustRegion(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
         @test vec(solve(prob, alg).u) ≈ solve(vecprob, alg).u
     end
 end
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index ca0e26ef6..1ef0757b8 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -1,7 +1,7 @@
 @testsetup module RootfindingTesting
 using Reexport
-@reexport using AllocCheck,
-                LinearSolve, StaticArrays, Random, LinearAlgebra, ForwardDiff, DiffEqBase
+@reexport using AllocCheck, LinearSolve, StaticArrays, Random, LinearAlgebra, ForwardDiff,
+                DiffEqBase
 import PolyesterForwardDiff
 
 quadratic_f(u, p) = u .* u .- p
@@ -14,16 +14,14 @@ function newton_fails(u, p)
             (0.21640425613334457 .+
              216.40425613334457 ./ (1 .+
               (0.21640425613334457 .+
-               216.40425613334457 ./
-               (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ 2.0) .-
-           0.0011552453009332421u .- p
+               216.40425613334457 ./ (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^
+            2.0) .- 0.0011552453009332421u .- p
 end
 
 const TERMINATION_CONDITIONS = [
     NormTerminationMode(), RelTerminationMode(), RelNormTerminationMode(),
     AbsTerminationMode(), AbsNormTerminationMode(), RelSafeTerminationMode(),
-    AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode()
-]
+    AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode()]
 
 function benchmark_nlsolve_oop(f::F, u0, p = 2.0; solver) where {F}
     prob = NonlinearProblem{false}(f, u0, p)
@@ -40,14 +38,14 @@ export quadratic_f, quadratic_f!, quadratic_f2, newton_fails, TERMINATION_CONDIT
 end
 
 @testitem "First Order Methods" setup=[RootfindingTesting] tags=[:core] begin
-    @testset "$(alg)" for alg in (SimpleNewtonRaphson, SimpleTrustRegion,
-        (args...; kwargs...) -> SimpleTrustRegion(args...; nlsolve_update_rule = Val(true),
-            kwargs...))
+    @testset "$(alg)" for alg in (SimpleNewtonRaphson,
+        SimpleTrustRegion,
+        (args...; kwargs...) -> SimpleTrustRegion(
+            args...; nlsolve_update_rule = Val(true), kwargs...))
         @testset "AutoDiff: $(nameof(typeof(autodiff))))" for autodiff in (
-            AutoFiniteDiff(),
-            AutoForwardDiff(), AutoPolyesterForwardDiff())
-            @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],
-                @SVector[1.0, 1.0], 1.0)
+            AutoFiniteDiff(), AutoForwardDiff(), AutoPolyesterForwardDiff())
+            @testset "[OOP] u0: $(typeof(u0))" for u0 in (
+                [1.0, 1.0], @SVector[1.0, 1.0], 1.0)
                 u0 isa SVector && autodiff isa AutoPolyesterForwardDiff && continue
                 sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = alg(; autodiff))
                 @test SciMLBase.successful_retcode(sol)
@@ -71,10 +69,10 @@ end
 end
 
 @testitem "SimpleHalley" setup=[RootfindingTesting] tags=[:core] begin
-    @testset "AutoDiff: $(nameof(typeof(autodiff)))" for autodiff in (AutoFiniteDiff(),
-        AutoForwardDiff())
-        @testset "[OOP] u0: $(nameof(typeof(u0)))" for u0 in ([1.0, 1.0],
-            @SVector[1.0, 1.0], 1.0)
+    @testset "AutoDiff: $(nameof(typeof(autodiff)))" for autodiff in (
+        AutoFiniteDiff(), AutoForwardDiff())
+        @testset "[OOP] u0: $(nameof(typeof(u0)))" for u0 in (
+            [1.0, 1.0], @SVector[1.0, 1.0], 1.0)
             sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = SimpleHalley(; autodiff))
             @test SciMLBase.successful_retcode(sol)
             @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
@@ -90,9 +88,9 @@ end
 end
 
 @testitem "Derivative Free Metods" setup=[RootfindingTesting] tags=[:core] begin
-    @testset "$(nameof(typeof(alg)))" for alg in [SimpleBroyden(), SimpleKlement(),
-        SimpleDFSane(), SimpleLimitedMemoryBroyden(),
-        SimpleBroyden(; linesearch = Val(true)),
+    @testset "$(nameof(typeof(alg)))" for alg in [
+        SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
+        SimpleLimitedMemoryBroyden(), SimpleBroyden(; linesearch = Val(true)),
         SimpleLimitedMemoryBroyden(; linesearch = Val(true))]
         @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
             sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = alg)
@@ -119,8 +117,9 @@ end
     u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
     p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
 
-    @testset "$(nameof(typeof(alg)))" for alg in (SimpleDFSane(), SimpleTrustRegion(),
-        SimpleHalley(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleDFSane(), SimpleTrustRegion(), SimpleHalley(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
         sol = benchmark_nlsolve_oop(newton_fails, u0, p; solver = alg)
         @test SciMLBase.successful_retcode(sol)
         @test all(abs.(newton_fails(sol.u, p)) .< 1e-9)
@@ -135,9 +134,10 @@ end
 
 @testitem "Allocation Checks" setup=[RootfindingTesting] tags=[:core] begin
     if Sys.islinux()  # Very slow on other OS
-        @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(),
-            SimpleHalley(), SimpleBroyden(), SimpleKlement(), SimpleLimitedMemoryBroyden(),
-            SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+        @testset "$(nameof(typeof(alg)))" for alg in (
+            SimpleNewtonRaphson(), SimpleHalley(), SimpleBroyden(),
+            SimpleKlement(), SimpleLimitedMemoryBroyden(), SimpleTrustRegion(),
+            SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
             SimpleDFSane(), SimpleBroyden(; linesearch = Val(true)),
             SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
             @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
@@ -166,8 +166,8 @@ end
 end
 
 @testitem "Interval Nonlinear Problems" setup=[RootfindingTesting] tags=[:core] begin
-    @testset "$(nameof(typeof(alg)))" for alg in (Bisection(), Falsi(), Ridder(), Brent(),
-        ITP(), Alefeld())
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        Bisection(), Falsi(), Ridder(), Brent(), ITP(), Alefeld())
         tspan = (1.0, 20.0)
 
         function g(p)
@@ -240,8 +240,8 @@ end
 end
 
 @testitem "Flipped Signs and Reversed Tspan" setup=[RootfindingTesting] tags=[:core] begin
-    @testset "$(nameof(typeof(alg)))" for alg in (Alefeld(), Bisection(), Falsi(), Brent(),
-        ITP(), Ridder())
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        Alefeld(), Bisection(), Falsi(), Brent(), ITP(), Ridder())
         f1(u, p) = u * u - p
         f2(u, p) = p - u * u
 
diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
index 39ac422a4..efc03403a 100644
--- a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
@@ -6,8 +6,9 @@
     f(u, p) = u .* u .- 2
     f!(du, u, p) = du .= u .* u .- 2
 
-    @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
-        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
         SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
         SimpleHalley(), SimpleBroyden(; linesearch = Val(true)),
         SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
@@ -51,10 +52,11 @@ end
 
     prob = NonlinearProblem{false}(f, @SVector[1.0f0, 1.0f0])
 
-    @testset "$(nameof(typeof(alg)))" for alg in (SimpleNewtonRaphson(), SimpleDFSane(),
-        SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
-        SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(), SimpleHalley(),
-        SimpleBroyden(; linesearch = Val(true)),
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+        SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
+        SimpleHalley(), SimpleBroyden(; linesearch = Val(true)),
         SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
         @test begin
             try

From efa09d8714c3e1ac39ee5b8dc89aa0281a2476b8 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 14:43:04 -0700
Subject: [PATCH 335/375] Add explicit imports

---
 lib/SimpleNonlinearSolve/Project.toml         |  4 +-
 .../ext/SimpleNonlinearSolveReverseDiffExt.jl | 42 +++++++++----------
 .../ext/SimpleNonlinearSolveTrackerExt.jl     |  2 +-
 .../src/SimpleNonlinearSolve.jl               | 36 ++++++++++------
 .../test/core/aqua_tests.jl                   |  9 ----
 .../test/core/qa_tests.jl                     | 23 ++++++++++
 6 files changed, 69 insertions(+), 47 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/test/core/aqua_tests.jl
 create mode 100644 lib/SimpleNonlinearSolve/test/core/qa_tests.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 74b6cde87..0fbdfd614 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -45,6 +45,7 @@ ChainRulesCore = "1.22"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.149"
 DiffResults = "1.1"
+ExplicitImports = "1.5.0"
 FastClosures = "0.3.2"
 FiniteDiff = "2.22"
 ForwardDiff = "0.10.36"
@@ -73,6 +74,7 @@ AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -91,4 +93,4 @@ Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["Aqua", "AllocCheck", "DiffEqBase", "ForwardDiff", "LinearAlgebra", "LinearSolve", "NonlinearProblemLibrary", "Pkg", "Random", "ReTestItems", "SciMLSensitivity", "StaticArrays", "Zygote", "CUDA", "PolyesterForwardDiff", "Reexport", "Test", "FiniteDiff", "ReverseDiff", "Tracker"]
+test = ["AllocCheck", "Aqua", "CUDA", "DiffEqBase", "ExplicitImports", "FiniteDiff", "ForwardDiff", "LinearAlgebra", "LinearSolve", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReTestItems", "Reexport", "ReverseDiff", "SciMLSensitivity", "StaticArrays", "Test", "Tracker", "Zygote"]
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
index a6a1c2dbf..c5f0286f1 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
@@ -5,65 +5,61 @@ using DiffEqBase: DiffEqBase
 using ReverseDiff: ReverseDiff, TrackedArray, TrackedReal
 using SciMLBase: ReverseDiffOriginator, NonlinearProblem, NonlinearLeastSquaresProblem
 using SimpleNonlinearSolve: SimpleNonlinearSolve
+import SimpleNonlinearSolve: __internal_solve_up
 
-function SimpleNonlinearSolve.__internal_solve_up(
+function __internal_solve_up(
         prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
         u0::TrackedArray, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
-        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
-function SimpleNonlinearSolve.__internal_solve_up(
+function __internal_solve_up(
         prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
         u0, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
-        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
-function SimpleNonlinearSolve.__internal_solve_up(
+function __internal_solve_up(
         prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
         u0::TrackedArray, u0_changed, p, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
-        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
 end
 
-function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+function __internal_solve_up(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
         sensealg, u0::AbstractArray{<:TrackedReal}, u0_changed,
         p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
-    return SimpleNonlinearSolve.__internal_solve_up(
-        prob, sensealg, ArrayInterface.aos_to_soa(u0), true,
+    return __internal_solve_up(prob, sensealg, ArrayInterface.aos_to_soa(u0), true,
         ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
 end
 
-function SimpleNonlinearSolve.__internal_solve_up(
+function __internal_solve_up(
         prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0,
         u0_changed, p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
-    return SimpleNonlinearSolve.__internal_solve_up(
-        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
+    return __internal_solve_up(prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
         true, alg, args...; kwargs...)
 end
 
-function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+function __internal_solve_up(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
         sensealg, u0::AbstractArray{<:TrackedReal},
         u0_changed, p, p_changed, alg, args...; kwargs...)
-    return SimpleNonlinearSolve.__internal_solve_up(
-        prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
+    return __internal_solve_up(prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
         true, alg, args...; kwargs...)
 end
 
-ReverseDiff.@grad function SimpleNonlinearSolve.__internal_solve_up(
+ReverseDiff.@grad function __internal_solve_up(
         prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
         sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...)
     out, ∇internal = DiffEqBase._solve_adjoint(
         prob, sensealg, ReverseDiff.value(u0), ReverseDiff.value(p),
         ReverseDiffOriginator(), alg, args...; kwargs...)
-    function ∇SimpleNonlinearSolve.__internal_solve_up(_args...)
+    function ∇__internal_solve_up(_args...)
         ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(_args...)
         return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
     end
-    return Array(out), ∇SimpleNonlinearSolve.__internal_solve_up
+    return Array(out), ∇__internal_solve_up
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
index b49bd78cc..85b84f80f 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
@@ -32,7 +32,7 @@ Tracker.@grad function SimpleNonlinearSolve.__internal_solve_up(
     u0, p = Tracker.data(u0_), Tracker.data(p_)
     prob = remake(_prob; u0, p)
     out, ∇internal = DiffEqBase._solve_adjoint(
-        prob, sensealg, u0, p, SciMLBase.TrackerOriginator(), alg, args...; kwargs...)
+        prob, sensealg, u0, p, TrackerOriginator(), alg, args...; kwargs...)
 
     function ∇__internal_solve_up(Δ)
         ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index dfd650c4d..1ff66b995 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -1,22 +1,31 @@
 module SimpleNonlinearSolve
 
-import PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidations
+using PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidations
 
 @recompile_invalidations begin
-    using ADTypes, ArrayInterface, ConcreteStructs, DiffEqBase, FastClosures, FiniteDiff,
-          ForwardDiff, Reexport, LinearAlgebra, SciMLBase
-
-    import DiffEqBase: AbstractNonlinearTerminationMode,
-                       AbstractSafeNonlinearTerminationMode,
-                       AbstractSafeBestNonlinearTerminationMode, NONLINEARSOLVE_DEFAULT_NORM
-    import DiffResults
-    import ForwardDiff: Dual
-    import MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
-    import SciMLBase: AbstractNonlinearAlgorithm, build_solution, isinplace, _unwrap_val
-    import StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
+    using ADTypes: ADTypes, AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff
+    using ArrayInterface: ArrayInterface
+    using ConcreteStructs: @concrete
+    using DiffEqBase: DiffEqBase, AbstractNonlinearTerminationMode,
+                      AbstractSafeNonlinearTerminationMode,
+                      AbstractSafeBestNonlinearTerminationMode, AbsNormTerminationMode,
+                      NONLINEARSOLVE_DEFAULT_NORM
+    using DiffResults: DiffResults
+    using FastClosures: @closure
+    using FiniteDiff: FiniteDiff
+    using ForwardDiff: ForwardDiff, Dual
+    using LinearAlgebra: LinearAlgebra, I, convert, copyto!, diagind, dot, issuccess, lu,
+                         mul!, norm, transpose
+    using MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
+    using Reexport: @reexport
+    using SciMLBase: SciMLBase, IntervalNonlinearProblem, NonlinearFunction,
+                     NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode, init,
+                     remake, solve, AbstractNonlinearAlgorithm, build_solution, isinplace,
+                     _unwrap_val
+    using StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
 end
 
-@reexport using ADTypes, SciMLBase
+@reexport using SciMLBase
 
 abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorithm end
 abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
@@ -110,6 +119,7 @@ end
     end
 end
 
+export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff
 export SimpleBroyden, SimpleDFSane, SimpleGaussNewton, SimpleHalley, SimpleKlement,
        SimpleLimitedMemoryBroyden, SimpleNewtonRaphson, SimpleTrustRegion
 export Alefeld, Bisection, Brent, Falsi, ITP, Ridder
diff --git a/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl b/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl
deleted file mode 100644
index 364f51b59..000000000
--- a/lib/SimpleNonlinearSolve/test/core/aqua_tests.jl
+++ /dev/null
@@ -1,9 +0,0 @@
-@testitem "Aqua" tags=[:core] begin
-    using Aqua
-
-    Aqua.test_all(SimpleNonlinearSolve; piracies = false, ambiguities = false)
-    Aqua.test_piracies(SimpleNonlinearSolve;
-        treat_as_own = [
-            NonlinearProblem, NonlinearLeastSquaresProblem, IntervalNonlinearProblem])
-    Aqua.test_ambiguities(SimpleNonlinearSolve; recursive = false)
-end
diff --git a/lib/SimpleNonlinearSolve/test/core/qa_tests.jl b/lib/SimpleNonlinearSolve/test/core/qa_tests.jl
new file mode 100644
index 000000000..fbdb813ee
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/qa_tests.jl
@@ -0,0 +1,23 @@
+@testitem "Aqua" tags=[:core] begin
+    using Aqua
+
+    Aqua.test_all(SimpleNonlinearSolve; piracies = false, ambiguities = false)
+    Aqua.test_piracies(SimpleNonlinearSolve;
+        treat_as_own = [
+            NonlinearProblem, NonlinearLeastSquaresProblem, IntervalNonlinearProblem])
+    Aqua.test_ambiguities(SimpleNonlinearSolve; recursive = false)
+end
+
+@testitem "Explicit Imports" tags=[:core] begin
+    import PolyesterForwardDiff, ReverseDiff, Tracker, StaticArrays, Zygote
+
+    using ExplicitImports
+
+    @test check_no_implicit_imports(
+        SimpleNonlinearSolve; skip = (SimpleNonlinearSolve, Base, Core, SciMLBase)) ===
+          nothing
+
+    @test check_no_stale_explicit_imports(SimpleNonlinearSolve) === nothing
+
+    @test check_all_qualified_accesses_via_owners(SimpleNonlinearSolve) === nothing
+end

From 248088c258d39bd7adf2801ab8240f17d7295436 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 15:01:59 -0700
Subject: [PATCH 336/375] Resolve ambiguity

---
 .../ext/SimpleNonlinearSolveReverseDiffExt.jl | 94 ++++++++++---------
 .../ext/SimpleNonlinearSolveTrackerExt.jl     | 74 ++++++++-------
 .../test/core/exotic_type_tests.jl            |  3 +-
 3 files changed, 88 insertions(+), 83 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
index c5f0286f1..249bbbedb 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
@@ -7,59 +7,61 @@ using SciMLBase: ReverseDiffOriginator, NonlinearProblem, NonlinearLeastSquaresP
 using SimpleNonlinearSolve: SimpleNonlinearSolve
 import SimpleNonlinearSolve: __internal_solve_up
 
-function __internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
-        u0::TrackedArray, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
-        u0_changed, p, p_changed, alg, args...; kwargs...)
-end
+for pType in (NonlinearProblem, NonlinearLeastSquaresProblem)
+    @eval begin
+        function __internal_solve_up(prob::$(pType), sensealg, u0::TrackedArray, u0_changed,
+                p::TrackedArray, p_changed, alg, args...; kwargs...)
+            return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+                u0_changed, p, p_changed, alg, args...; kwargs...)
+        end
 
-function __internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
-        u0, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
-        u0_changed, p, p_changed, alg, args...; kwargs...)
-end
+        function __internal_solve_up(prob::$(pType), sensealg, u0, u0_changed,
+                p::TrackedArray, p_changed, alg, args...; kwargs...)
+            return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+                u0_changed, p, p_changed, alg, args...; kwargs...)
+        end
 
-function __internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
-        u0::TrackedArray, u0_changed, p, p_changed, alg, args...; kwargs...)
-    return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
-        u0_changed, p, p_changed, alg, args...; kwargs...)
-end
+        function __internal_solve_up(prob::$(pType), sensealg, u0::TrackedArray,
+                u0_changed, p, p_changed, alg, args...; kwargs...)
+            return ReverseDiff.track(__internal_solve_up, prob, sensealg, u0,
+                u0_changed, p, p_changed, alg, args...; kwargs...)
+        end
 
-function __internal_solve_up(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
-        sensealg, u0::AbstractArray{<:TrackedReal}, u0_changed,
-        p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
-    return __internal_solve_up(prob, sensealg, ArrayInterface.aos_to_soa(u0), true,
-        ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
-end
+        function __internal_solve_up(
+                prob::$(pType), sensealg, u0::AbstractArray{<:TrackedReal}, u0_changed,
+                p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
+            return __internal_solve_up(prob, sensealg, ArrayInterface.aos_to_soa(u0), true,
+                ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
+        end
 
-function __internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0,
-        u0_changed, p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
-    return __internal_solve_up(prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
-        true, alg, args...; kwargs...)
-end
+        function __internal_solve_up(prob::$(pType), sensealg, u0, u0_changed,
+                p::AbstractArray{<:TrackedReal}, p_changed, alg, args...; kwargs...)
+            return __internal_solve_up(
+                prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
+                true, alg, args...; kwargs...)
+        end
 
-function __internal_solve_up(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
-        sensealg, u0::AbstractArray{<:TrackedReal},
-        u0_changed, p, p_changed, alg, args...; kwargs...)
-    return __internal_solve_up(prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
-        true, alg, args...; kwargs...)
-end
+        function __internal_solve_up(
+                prob::$(pType), sensealg, u0::AbstractArray{<:TrackedReal},
+                u0_changed, p, p_changed, alg, args...; kwargs...)
+            return __internal_solve_up(
+                prob, sensealg, u0, true, ArrayInterface.aos_to_soa(p),
+                true, alg, args...; kwargs...)
+        end
 
-ReverseDiff.@grad function __internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
-        sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...)
-    out, ∇internal = DiffEqBase._solve_adjoint(
-        prob, sensealg, ReverseDiff.value(u0), ReverseDiff.value(p),
-        ReverseDiffOriginator(), alg, args...; kwargs...)
-    function ∇__internal_solve_up(_args...)
-        ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(_args...)
-        return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
+        ReverseDiff.@grad function __internal_solve_up(
+                prob::$(pType), sensealg, u0, u0_changed,
+                p, p_changed, alg, args...; kwargs...)
+            out, ∇internal = DiffEqBase._solve_adjoint(
+                prob, sensealg, ReverseDiff.value(u0), ReverseDiff.value(p),
+                ReverseDiffOriginator(), alg, args...; kwargs...)
+            function ∇__internal_solve_up(_args...)
+                ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(_args...)
+                return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
+            end
+            return Array(out), ∇__internal_solve_up
+        end
     end
-    return Array(out), ∇__internal_solve_up
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
index 85b84f80f..a212b220e 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
@@ -1,45 +1,49 @@
 module SimpleNonlinearSolveTrackerExt
 
 using DiffEqBase: DiffEqBase
-using SciMLBase: TrackerOriginator, NonlinearProblem, NonlinearLeastSquaresProblem
+using SciMLBase: TrackerOriginator, NonlinearProblem, NonlinearLeastSquaresProblem, remake
 using SimpleNonlinearSolve: SimpleNonlinearSolve
 using Tracker: Tracker, TrackedArray
 
-function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
-        u0::TrackedArray, u0_changed, p, p_changed, alg, args...; kwargs...)
-    return Tracker.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
-        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
-end
-
-function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
-        u0::TrackedArray, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return Tracker.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
-        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
-end
-
-function SimpleNonlinearSolve.__internal_solve_up(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, sensealg,
-        u0, u0_changed, p::TrackedArray, p_changed, alg, args...; kwargs...)
-    return Tracker.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
-        u0, u0_changed, p, p_changed, alg, args...; kwargs...)
-end
-
-Tracker.@grad function SimpleNonlinearSolve.__internal_solve_up(
-        _prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
-        sensealg, u0_, u0_changed, p_, p_changed, alg, args...; kwargs...)
-    u0, p = Tracker.data(u0_), Tracker.data(p_)
-    prob = remake(_prob; u0, p)
-    out, ∇internal = DiffEqBase._solve_adjoint(
-        prob, sensealg, u0, p, TrackerOriginator(), alg, args...; kwargs...)
-
-    function ∇__internal_solve_up(Δ)
-        ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
-        return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
+for pType in (NonlinearProblem, NonlinearLeastSquaresProblem)
+    @eval begin
+        function SimpleNonlinearSolve.__internal_solve_up(
+                prob::$(pType), sensealg, u0::TrackedArray,
+                u0_changed, p, p_changed, alg, args...; kwargs...)
+            return Tracker.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
+                u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+        end
+
+        function SimpleNonlinearSolve.__internal_solve_up(
+                prob::$(pType), sensealg, u0::TrackedArray, u0_changed,
+                p::TrackedArray, p_changed, alg, args...; kwargs...)
+            return Tracker.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
+                u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+        end
+
+        function SimpleNonlinearSolve.__internal_solve_up(
+                prob::$(pType), sensealg, u0, u0_changed,
+                p::TrackedArray, p_changed, alg, args...; kwargs...)
+            return Tracker.track(SimpleNonlinearSolve.__internal_solve_up, prob, sensealg,
+                u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+        end
+
+        Tracker.@grad function SimpleNonlinearSolve.__internal_solve_up(
+                _prob::$(pType), sensealg, u0_, u0_changed,
+                p_, p_changed, alg, args...; kwargs...)
+            u0, p = Tracker.data(u0_), Tracker.data(p_)
+            prob = remake(_prob; u0, p)
+            out, ∇internal = DiffEqBase._solve_adjoint(
+                prob, sensealg, u0, p, TrackerOriginator(), alg, args...; kwargs...)
+
+            function ∇__internal_solve_up(Δ)
+                ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
+                return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
+            end
+
+            return out, ∇__internal_solve_up
+        end
     end
-
-    return out, ∇__internal_solve_up
 end
 
 end
diff --git a/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl
index fff77a3d4..302d2402e 100644
--- a/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl
@@ -16,8 +16,7 @@ end
     using SimpleNonlinearSolve, LinearAlgebra
 
     for alg in [SimpleNewtonRaphson(), SimpleBroyden(), SimpleKlement(), SimpleDFSane(),
-        SimpleTrustRegion(), SimpleLimitedMemoryBroyden(; threshold = 2),
-        SimpleHalley()]
+        SimpleTrustRegion(), SimpleLimitedMemoryBroyden(; threshold = 2), SimpleHalley()]
         sol = solve(prob_oop_bf, alg)
         @test norm(sol.resid, Inf) < 1e-6
         @test SciMLBase.successful_retcode(sol.retcode)

From b84ee8f5332ccf83f8ed6198850430ba39f2eb79 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 18:12:57 -0700
Subject: [PATCH 337/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 0fbdfd614..7023d27d6 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.8.2"
+version = "1.8.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From 02160aa00abac72946cd97e20f86832e0111dd62 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 16:29:04 -0700
Subject: [PATCH 338/375] Use DifferentiationInterface

---
 lib/SimpleNonlinearSolve/Project.toml         |   6 +-
 ...leNonlinearSolvePolyesterForwardDiffExt.jl |  20 --
 .../src/SimpleNonlinearSolve.jl               |   6 +-
 .../src/nlsolve/halley.jl                     |  14 +-
 .../src/nlsolve/raphson.jl                    |  11 +-
 .../src/nlsolve/trustRegion.jl                |  13 +-
 lib/SimpleNonlinearSolve/src/utils.jl         | 238 ++++--------------
 7 files changed, 78 insertions(+), 230 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 7023d27d6..5284b3ed4 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.8.1"
+version = "1.9.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -9,6 +9,7 @@ ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
 ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 DiffResults = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
 FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
@@ -21,7 +22,6 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [weakdeps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
 ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
@@ -29,7 +29,6 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [extensions]
 SimpleNonlinearSolveChainRulesCoreExt = "ChainRulesCore"
-SimpleNonlinearSolvePolyesterForwardDiffExt = "PolyesterForwardDiff"
 SimpleNonlinearSolveReverseDiffExt = "ReverseDiff"
 SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
 SimpleNonlinearSolveTrackerExt = "Tracker"
@@ -45,6 +44,7 @@ ChainRulesCore = "1.22"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.149"
 DiffResults = "1.1"
+DifferentiationInterface = "0.4"
 ExplicitImports = "1.5.0"
 FastClosures = "0.3.2"
 FiniteDiff = "2.22"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
deleted file mode 100644
index ac898ac16..000000000
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolvePolyesterForwardDiffExt.jl
+++ /dev/null
@@ -1,20 +0,0 @@
-module SimpleNonlinearSolvePolyesterForwardDiffExt
-
-using PolyesterForwardDiff: PolyesterForwardDiff
-using SimpleNonlinearSolve: SimpleNonlinearSolve
-
-@inline SimpleNonlinearSolve.__is_extension_loaded(::Val{:PolyesterForwardDiff}) = true
-
-@inline function SimpleNonlinearSolve.__polyester_forwarddiff_jacobian!(
-        f!::F, y, J, x, chunksize) where {F}
-    PolyesterForwardDiff.threaded_jacobian!(f!, y, J, x, chunksize)
-    return J
-end
-
-@inline function SimpleNonlinearSolve.__polyester_forwarddiff_jacobian!(
-        f::F, J, x, chunksize) where {F}
-    PolyesterForwardDiff.threaded_jacobian!(f, J, x, chunksize)
-    return J
-end
-
-end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 1ff66b995..a024f069e 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -3,13 +3,15 @@ module SimpleNonlinearSolve
 using PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidations
 
 @recompile_invalidations begin
-    using ADTypes: ADTypes, AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff
+    using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff,
+                   AutoPolyesterForwardDiff
     using ArrayInterface: ArrayInterface
     using ConcreteStructs: @concrete
     using DiffEqBase: DiffEqBase, AbstractNonlinearTerminationMode,
                       AbstractSafeNonlinearTerminationMode,
                       AbstractSafeBestNonlinearTerminationMode, AbsNormTerminationMode,
                       NONLINEARSOLVE_DEFAULT_NORM
+    using DifferentiationInterface: DifferentiationInterface
     using DiffResults: DiffResults
     using FastClosures: @closure
     using FiniteDiff: FiniteDiff
@@ -25,6 +27,8 @@ using PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidati
     using StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
 end
 
+const DI = DifferentiationInterface
+
 @reexport using SciMLBase
 
 abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorithm end
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index e550258ef..5ff5eea40 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -12,8 +12,9 @@ A low-overhead implementation of Halley's Method.
 
 ### Keyword Arguments
 
-  - `autodiff`: determines the backend used for the Hessian. Defaults to `nothing`. Valid
-    choices are `AutoForwardDiff()` or `AutoFiniteDiff()`.
+  - `autodiff`: determines the backend used for the Hessian. Defaults to `nothing` (i.e.
+    automatic backend selection). Valid choices include backends from
+    `DifferentiationInterface.jl`.
 
 !!! warning
 
@@ -26,13 +27,11 @@ end
 function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         alias_u0 = false, termination_condition = nothing, kwargs...)
-    isinplace(prob) &&
-        error("SimpleHalley currently only supports out-of-place nonlinear problems")
-
     x = __maybe_unaliased(prob.u0, alias_u0)
     fx = _get_fx(prob, x)
     T = eltype(x)
 
+    f = __fixed_parameter_function(prob)
     autodiff = __get_concrete_autodiff(prob, alg.autodiff)
     abstol, reltol, tc_cache = init_termination_cache(
         prob, abstol, reltol, fx, x, termination_condition)
@@ -51,7 +50,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
 
     for i in 1:maxiters
         # Hessian Computation is unfortunately type unstable
-        fx, dfx, d2fx = compute_jacobian_and_hessian(autodiff, prob, fx, x)
+        fx, dfx, d2fx = compute_jacobian_and_hessian(autodiff, prob, f, fx, x)
         setindex_trait(x) === CannotSetindex() && (A = dfx)
 
         # Factorize Once and Reuse
@@ -78,9 +77,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
         cᵢ = _restructure(cᵢ, cᵢ_)
 
         if i == 1
-            if iszero(fx)
+            iszero(fx) &&
                 return build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
-            end
         else
             # Termination Checks
             tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
index 7b419ce99..ca6864b28 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
@@ -13,9 +13,9 @@ and static array problems.
 
 ### Keyword Arguments
 
-  - `autodiff`: determines the backend used for the Jacobian. Defaults to
-    `nothing`. Valid choices are `AutoPolyesterForwardDiff()`, `AutoForwardDiff()` or
-    `AutoFiniteDiff()`.
+  - `autodiff`: determines the backend used for the Jacobian. Defaults to  `nothing` (i.e.
+    automatic backend selection). Valid choices include jacobian backends from
+    `DifferentiationInterface.jl`.
 """
 @kwdef @concrete struct SimpleNewtonRaphson <: AbstractNewtonAlgorithm
     autodiff = nothing
@@ -30,13 +30,14 @@ function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresPr
     fx = _get_fx(prob, x)
     autodiff = __get_concrete_autodiff(prob, alg.autodiff)
     @bb xo = copy(x)
-    J, jac_cache = jacobian_cache(autodiff, prob.f, fx, x, prob.p)
+    f = __fixed_parameter_function(prob)
+    J, jac_cache = jacobian_cache(autodiff, prob, f, fx, x)
 
     abstol, reltol, tc_cache = init_termination_cache(
         prob, abstol, reltol, fx, x, termination_condition)
 
     for i in 1:maxiters
-        fx, dfx = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
+        fx, dfx = value_and_jacobian(autodiff, prob, f, fx, x, jac_cache; J)
 
         if i == 1
             iszero(fx) && build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index a19cf2c2d..6ff263ecd 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -10,9 +10,9 @@ scalar and static array problems.
 
 ### Keyword Arguments
 
-  - `autodiff`: determines the backend used for the Jacobian. Defaults to
-    `nothing`. Valid choices are `AutoPolyesterForwardDiff()`, `AutoForwardDiff()` or
-    `AutoFiniteDiff()`.
+  - `autodiff`: determines the backend used for the Jacobian. Defaults to `nothing` (i.e.
+    automatic backend selection). Valid choices include jacobian backends from
+    `DifferentiationInterface.jl`.
   - `max_trust_radius`: the maximum radius of the trust region. Defaults to
     `max(norm(f(u0)), maximum(u0) - minimum(u0))`.
   - `initial_trust_radius`: the initial trust region radius. Defaults to
@@ -85,8 +85,9 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
     fx = _get_fx(prob, x)
     norm_fx = norm(fx)
     @bb xo = copy(x)
-    J, jac_cache = jacobian_cache(autodiff, prob.f, fx, x, prob.p)
-    fx, ∇f = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
+    f = __fixed_parameter_function(prob)
+    J, jac_cache = jacobian_cache(autodiff, prob, f, fx, x)
+    fx, ∇f = value_and_jacobian(autodiff, prob, f, fx, x, jac_cache; J)
 
     abstol, reltol, tc_cache = init_termination_cache(
         prob, abstol, reltol, fx, x, termination_condition)
@@ -144,7 +145,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args.
             # Take the step.
             @bb @. xo = x
 
-            fx, ∇f = value_and_jacobian(autodiff, prob.f, fx, x, prob.p, jac_cache; J)
+            fx, ∇f = value_and_jacobian(autodiff, prob, f, fx, x, jac_cache; J)
 
             # Update the trust region radius.
             if !_unwrap_val(alg.nlsolve_update_rule) && r > η₃
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 333e54d3b..ae380d7be 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -1,9 +1,4 @@
-struct SimpleNonlinearSolveTag end
-
-function ForwardDiff.checktag(::Type{<:ForwardDiff.Tag{<:SimpleNonlinearSolveTag, <:T}},
-        f::F, x::AbstractArray{T}) where {T, F}
-    return true
-end
+struct HasAnalyticJacobian end
 
 """
     __prevfloat_tdir(x, x0, x1)
@@ -26,203 +21,63 @@ Return the maximum of `a` and `b` if `x1 > x0`, otherwise return the minimum.
 """
 __max_tdir(a, b, x0, x1) = ifelse(x1 > x0, max(a, b), min(a, b))
 
-__standard_tag(::Nothing, x) = ForwardDiff.Tag(SimpleNonlinearSolveTag(), eltype(x))
-__standard_tag(tag::ForwardDiff.Tag, _) = tag
-__standard_tag(tag, x) = ForwardDiff.Tag(tag, eltype(x))
-
-__pick_forwarddiff_chunk(x) = ForwardDiff.Chunk(length(x))
-function __pick_forwarddiff_chunk(x::StaticArray)
-    L = prod(Size(x))
-    if L ≤ ForwardDiff.DEFAULT_CHUNK_THRESHOLD
-        return ForwardDiff.Chunk{L}()
-    else
-        return ForwardDiff.Chunk{ForwardDiff.DEFAULT_CHUNK_THRESHOLD}()
-    end
-end
-
-function __get_jacobian_config(ad::AutoForwardDiff{CS}, f::F, x) where {F, CS}
-    ck = (CS === nothing || CS ≤ 0) ? __pick_forwarddiff_chunk(x) : ForwardDiff.Chunk{CS}()
-    tag = __standard_tag(ad.tag, x)
-    return __forwarddiff_jacobian_config(f, x, ck, tag)
-end
-function __get_jacobian_config(ad::AutoForwardDiff{CS}, f!::F, y, x) where {F, CS}
-    ck = (CS === nothing || CS ≤ 0) ? __pick_forwarddiff_chunk(x) : ForwardDiff.Chunk{CS}()
-    tag = __standard_tag(ad.tag, x)
-    return ForwardDiff.JacobianConfig(f!, y, x, ck, tag)
-end
-
-function __forwarddiff_jacobian_config(f::F, x, ck::ForwardDiff.Chunk, tag) where {F}
-    return ForwardDiff.JacobianConfig(f, x, ck, tag)
-end
-function __forwarddiff_jacobian_config(
-        f::F, x::SArray, ck::ForwardDiff.Chunk{N}, tag) where {F, N}
-    seeds = ForwardDiff.construct_seeds(ForwardDiff.Partials{N, eltype(x)})
-    duals = ForwardDiff.Dual{typeof(tag), eltype(x), N}.(x)
-    return ForwardDiff.JacobianConfig{typeof(tag), eltype(x), N, typeof(duals)}(
-        seeds, duals)
+function __fixed_parameter_function(prob::NonlinearProblem)
+    isinplace(prob) && return @closure (du, u) -> prob.f(du, u, prob.p)
+    return Base.Fix2(prob.f, prob.p)
 end
 
-function __get_jacobian_config(ad::AutoPolyesterForwardDiff{CS}, args...) where {CS}
-    x = last(args)
-    return (CS === nothing || CS ≤ 0) ? __pick_forwarddiff_chunk(x) :
-           ForwardDiff.Chunk{CS}()
-end
+function value_and_jacobian(
+        ad, prob::NonlinearProblem, f::F, y, x, cache; J = nothing) where {F}
+    x isa Number && return DI.value_and_derivative(f, ad, x, cache)
 
-"""
-    value_and_jacobian(ad, f, y, x, p, cache; J = nothing)
-
-Compute `f(x), d/dx f(x)` in the most efficient way based on `ad`. None of the arguments
-except `cache` (& `J` if not nothing) are mutated.
-"""
-function value_and_jacobian(ad, f::F, y, x::X, p, cache; J = nothing) where {F, X}
-    if isinplace(f)
-        _f = (du, u) -> f(du, u, p)
-        if SciMLBase.has_jac(f)
-            f.jac(J, x, p)
-            _f(y, x)
-            return y, J
-        elseif ad isa AutoForwardDiff
-            res = DiffResults.DiffResult(y, J)
-            ForwardDiff.jacobian!(res, _f, y, x, cache)
-            return DiffResults.value(res), DiffResults.jacobian(res)
-        elseif ad isa AutoFiniteDiff
-            FiniteDiff.finite_difference_jacobian!(J, _f, x, cache)
-            _f(y, x)
-            return y, J
-        elseif ad isa AutoPolyesterForwardDiff
-            __polyester_forwarddiff_jacobian!(_f, y, J, x, cache)
-            return y, J
+    if isinplace(prob)
+        if cache isa HasAnalyticJacobian
+            prob.f.jac(J, x, p)
+            f(y, x)
         else
-            throw(ArgumentError("Unsupported AD method: $(ad)"))
+            DI.jacobian!(f, y, J, ad, x, cache)
         end
+        return y, J
     else
-        _f = Base.Fix2(f, p)
-        if SciMLBase.has_jac(f)
-            return _f(x), f.jac(x, p)
-        elseif ad isa AutoForwardDiff
-            if ArrayInterface.can_setindex(x)
-                res = DiffResults.DiffResult(y, J)
-                ForwardDiff.jacobian!(res, _f, x, cache)
-                return DiffResults.value(res), DiffResults.jacobian(res)
-            else
-                J_fd = ForwardDiff.jacobian(_f, x, cache)
-                return _f(x), J_fd
-            end
-        elseif ad isa AutoFiniteDiff
-            J_fd = FiniteDiff.finite_difference_jacobian(_f, x, cache)
-            return _f(x), J_fd
-        elseif ad isa AutoPolyesterForwardDiff
-            __polyester_forwarddiff_jacobian!(_f, J, x, cache)
-            return _f(x), J
-        else
-            throw(ArgumentError("Unsupported AD method: $(ad)"))
-        end
+        cache isa HasAnalyticJacobian && return f(x), prob.f.jac(x, prob.p)
+        J === nothing && return DI.value_and_jacobian(f, ad, x, cache)
+        y, _ = DI.value_and_jacobian!(f, J, ad, x, cache)
+        return y, J
     end
 end
 
-# Declare functions
-function __polyester_forwarddiff_jacobian! end
-
-function value_and_jacobian(ad, f::F, y, x::Number, p, cache; J = nothing) where {F}
-    if SciMLBase.has_jac(f)
-        return f(x, p), f.jac(x, p)
-    elseif ad isa AutoForwardDiff
-        T = typeof(__standard_tag(ad.tag, x))
-        out = f(ForwardDiff.Dual{T}(x, one(x)), p)
-        return ForwardDiff.value(out), ForwardDiff.extract_derivative(T, out)
-    elseif ad isa AutoPolyesterForwardDiff
-        # Just use ForwardDiff
-        T = typeof(__standard_tag(nothing, x))
-        out = f(ForwardDiff.Dual{T}(x, one(x)), p)
-        return ForwardDiff.value(out), ForwardDiff.extract_derivative(T, out)
-    elseif ad isa AutoFiniteDiff
-        _f = Base.Fix2(f, p)
-        return _f(x), FiniteDiff.finite_difference_derivative(_f, x, ad.fdtype)
-    else
-        throw(ArgumentError("Unsupported AD method: $(ad)"))
-    end
-end
+function jacobian_cache(ad, prob::NonlinearProblem, f::F, y, x) where {F}
+    x isa Number && return (nothing, DI.prepare_derivative(f, ad, x))
 
-"""
-    jacobian_cache(ad, f, y, x, p) --> J, cache
-
-Returns a Jacobian Matrix and a cache for the Jacobian computation.
-"""
-function jacobian_cache(ad, f::F, y, x::X, p) where {F, X <: AbstractArray}
-    if isinplace(f)
-        _f = (du, u) -> f(du, u, p)
+    if isinplace(prob)
         J = similar(y, length(y), length(x))
-        if SciMLBase.has_jac(f)
-            return J, nothing
-        elseif ad isa AutoForwardDiff || ad isa AutoPolyesterForwardDiff
-            return J, __get_jacobian_config(ad, _f, y, x)
-        elseif ad isa AutoFiniteDiff
-            return J, FiniteDiff.JacobianCache(copy(x), copy(y), copy(y), ad.fdtype)
-        else
-            throw(ArgumentError("Unsupported AD method: $(ad)"))
-        end
+        SciMLBase.has_jac(prob.f) && return J, HasAnalyticJacobian()
+        return J, DI.prepare_jacobian(f, y, ad, x)
     else
-        _f = Base.Fix2(f, p)
-        if SciMLBase.has_jac(f)
-            return nothing, nothing
-        elseif ad isa AutoForwardDiff
-            J = ArrayInterface.can_setindex(x) ? similar(y, length(y), length(x)) : nothing
-            return J, __get_jacobian_config(ad, _f, x)
-        elseif ad isa AutoPolyesterForwardDiff
-            @assert ArrayInterface.can_setindex(x) "PolyesterForwardDiff requires mutable inputs. Use AutoForwardDiff instead."
-            J = similar(y, length(y), length(x))
-            return J, __get_jacobian_config(ad, _f, x)
-        elseif ad isa AutoFiniteDiff
-            return nothing, FiniteDiff.JacobianCache(copy(x), copy(y), copy(y), ad.fdtype)
-        else
-            throw(ArgumentError("Unsupported AD method: $(ad)"))
-        end
+        SciMLBase.has_jac(prob.f) && return nothing, HasAnalyticJacobian()
+        J = ArrayInterface.can_setindex(x) ? similar(y, length(y), length(x)) : nothing
+        return J, DI.prepare_jacobian(f, ad, x)
     end
 end
 
-jacobian_cache(ad, f::F, y, x::Number, p) where {F} = nothing, nothing
-
-function compute_jacobian_and_hessian(ad::AutoForwardDiff, prob, _, x::Number)
-    fx = prob.f(x, prob.p)
-    J_fn = Base.Fix1(ForwardDiff.derivative, Base.Fix2(prob.f, prob.p))
-    dfx = J_fn(x)
-    d2fx = ForwardDiff.derivative(J_fn, x)
-    return fx, dfx, d2fx
-end
-
-function compute_jacobian_and_hessian(ad::AutoForwardDiff, prob, fx, x)
-    if isinplace(prob)
-        error("Inplace version for Nested ForwardDiff Not Implemented Yet!")
-    else
-        f = Base.Fix2(prob.f, prob.p)
-        fx = f(x)
-        J_fn = Base.Fix1(ForwardDiff.jacobian, f)
-        dfx = J_fn(x)
-        d2fx = ForwardDiff.jacobian(J_fn, x)
-        return fx, dfx, d2fx
+function compute_jacobian_and_hessian(ad, prob::NonlinearProblem, f::F, y, x) where {F}
+    if x isa Number
+        df = @closure x -> DI.derivative(f, ad, x)
+        return f(x), df(x), DI.derivative(df, ad, x)
     end
-end
-
-function compute_jacobian_and_hessian(ad::AutoFiniteDiff, prob, _, x::Number)
-    fx = prob.f(x, prob.p)
-    J_fn = x -> FiniteDiff.finite_difference_derivative(
-        Base.Fix2(prob.f, prob.p), x, ad.fdtype)
-    dfx = J_fn(x)
-    d2fx = FiniteDiff.finite_difference_derivative(J_fn, x, ad.fdtype)
-    return fx, dfx, d2fx
-end
 
-function compute_jacobian_and_hessian(ad::AutoFiniteDiff, prob, fx, x)
     if isinplace(prob)
-        error("Inplace version for Nested FiniteDiff Not Implemented Yet!")
-    else
-        f = Base.Fix2(prob.f, prob.p)
-        fx = f(x)
-        J_fn = x -> FiniteDiff.finite_difference_jacobian(f, x, ad.fdtype)
-        dfx = J_fn(x)
-        d2fx = FiniteDiff.finite_difference_jacobian(J_fn, x, ad.fdtype)
-        return fx, dfx, d2fx
+        df = @closure x -> begin
+            res = similar(y, promote_type(eltype(y), eltype(x)))
+            return DI.jacobian(f, res, ad, x)
+        end
+        J, H = DI.value_and_jacobian(df, ad, x)
+        f(y, x)
+        return y, J, H
     end
+
+    df = @closure x -> DI.jacobian(f, ad, x)
+    return f(x), df(x), DI.jacobian(df, ad, x)
 end
 
 __init_identity_jacobian(u::Number, fu, α = true) = oftype(u, α)
@@ -360,10 +215,19 @@ end
 end
 
 # Decide which AD backend to use
-@inline __get_concrete_autodiff(prob, ad::ADTypes.AbstractADType; kwargs...) = ad
+@inline __get_concrete_autodiff(prob, ad::AbstractADType; kwargs...) = ad
+@inline function __get_concrete_autodiff(prob, ad::AutoForwardDiff{nothing}; kwargs...)
+    return AutoForwardDiff(; chunksize = ForwardDiff.pickchunksize(length(prob.u0)), ad.tag)
+end
+@inline function __get_concrete_autodiff(
+        prob, ad::AutoPolyesterForwardDiff{nothing}; kwargs...)
+    return AutoPolyesterForwardDiff(;
+        chunksize = ForwardDiff.pickchunksize(length(prob.u0)), ad.tag)
+end
 @inline function __get_concrete_autodiff(prob, ::Nothing; kwargs...)
-    return ifelse(
-        ForwardDiff.can_dual(eltype(prob.u0)), AutoForwardDiff(), AutoFiniteDiff())
+    return ifelse(ForwardDiff.can_dual(eltype(prob.u0)),
+        AutoForwardDiff(; chunksize = ForwardDiff.pickchunksize(length(prob.u0))),
+        AutoFiniteDiff())
 end
 
 @inline __reshape(x::Number, args...) = x

From 3c32525d5a4ff382bef3091a640acbd484d6b904 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 17:31:06 -0700
Subject: [PATCH 339/375] Clean up AD

---
 lib/SimpleNonlinearSolve/Project.toml         |  2 +-
 .../src/SimpleNonlinearSolve.jl               |  8 +-
 lib/SimpleNonlinearSolve/src/ad.jl            | 92 +++++++------------
 .../src/nlsolve/halley.jl                     |  6 +-
 .../src/nlsolve/lbroyden.jl                   |  6 +-
 lib/SimpleNonlinearSolve/src/utils.jl         | 32 +++++--
 6 files changed, 68 insertions(+), 78 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 5284b3ed4..a7aa31f82 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -47,7 +47,7 @@ DiffResults = "1.1"
 DifferentiationInterface = "0.4"
 ExplicitImports = "1.5.0"
 FastClosures = "0.3.2"
-FiniteDiff = "2.22"
+FiniteDiff = "2.23.1"
 ForwardDiff = "0.10.36"
 LinearAlgebra = "1.10"
 LinearSolve = "2.30"
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index a024f069e..04a32c969 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -20,10 +20,10 @@ using PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidati
                          mul!, norm, transpose
     using MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
     using Reexport: @reexport
-    using SciMLBase: SciMLBase, IntervalNonlinearProblem, NonlinearFunction,
-                     NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode, init,
-                     remake, solve, AbstractNonlinearAlgorithm, build_solution, isinplace,
-                     _unwrap_val
+    using SciMLBase: SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem,
+                     NonlinearFunction, NonlinearLeastSquaresProblem, NonlinearProblem,
+                     ReturnCode, init, remake, solve, AbstractNonlinearAlgorithm,
+                     build_solution, isinplace, _unwrap_val
     using StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
 end
 
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index f42651bfe..ffae7cc27 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -1,25 +1,15 @@
-function SciMLBase.solve(
-        prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip,
-            <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
-        alg::AbstractSimpleNonlinearSolveAlgorithm,
-        args...;
-        kwargs...) where {T, V, P, iip}
-    sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
-    dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
-    return SciMLBase.build_solution(
-        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
-end
-
-function SciMLBase.solve(
-        prob::NonlinearLeastSquaresProblem{
-            <:AbstractArray, iip, <:Union{<:AbstractArray{<:Dual{T, V, P}}}},
-        alg::AbstractSimpleNonlinearSolveAlgorithm,
-        args...;
-        kwargs...) where {T, V, P, iip}
-    sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
-    dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
-    return SciMLBase.build_solution(
-        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
+for pType in (NonlinearProblem, NonlinearLeastSquaresProblem)
+    @eval function SciMLBase.solve(
+            prob::$(pType){<:Union{Number, <:AbstractArray}, iip,
+                <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
+            alg::AbstractSimpleNonlinearSolveAlgorithm,
+            args...;
+            kwargs...) where {T, V, P, iip}
+        sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
+        dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
+        return SciMLBase.build_solution(
+            prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
+    end
 end
 
 for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
@@ -47,8 +37,7 @@ function __nlsolve_ad(
         tspan = value.(prob.tspan)
         newprob = IntervalNonlinearProblem(prob.f, tspan, p; prob.kwargs...)
     else
-        u0 = value(prob.u0)
-        newprob = NonlinearProblem(prob.f, u0, p; prob.kwargs...)
+        newprob = remake(prob; p, u0 = value(prob.u0))
     end
 
     sol = solve(newprob, alg, args...; kwargs...)
@@ -73,12 +62,8 @@ function __nlsolve_ad(
 end
 
 function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs...)
-    p = value(prob.p)
-    u0 = value(prob.u0)
-    newprob = NonlinearLeastSquaresProblem(prob.f, u0, p; prob.kwargs...)
-
+    newprob = remake(prob; p = value(prob.p), u0 = value(prob.u0))
     sol = solve(newprob, alg, args...; kwargs...)
-
     uu = sol.u
 
     # First check for custom `vjp` then custom `Jacobian` and if nothing is provided use
@@ -86,7 +71,7 @@ function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs..
     if SciMLBase.has_vjp(prob.f)
         if isinplace(prob)
             _F = @closure (du, u, p) -> begin
-                resid = similar(du, length(sol.resid))
+                resid = __similar(du, length(sol.resid))
                 prob.f(resid, u, p)
                 prob.f.vjp(du, resid, u, p)
                 du .*= 2
@@ -101,9 +86,9 @@ function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs..
     elseif SciMLBase.has_jac(prob.f)
         if isinplace(prob)
             _F = @closure (du, u, p) -> begin
-                J = similar(du, length(sol.resid), length(u))
+                J = __similar(du, length(sol.resid), length(u))
                 prob.f.jac(J, u, p)
-                resid = similar(du, length(sol.resid))
+                resid = __similar(du, length(sol.resid))
                 prob.f(resid, u, p)
                 mul!(reshape(du, 1, :), vec(resid)', J, 2, false)
                 return nothing
@@ -116,35 +101,30 @@ function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs..
     else
         if isinplace(prob)
             _F = @closure (du, u, p) -> begin
-                resid = similar(du, length(sol.resid))
-                res = DiffResults.DiffResult(
-                    resid, similar(du, length(sol.resid), length(u)))
                 _f = @closure (du, u) -> prob.f(du, u, p)
-                ForwardDiff.jacobian!(res, _f, resid, u)
-                mul!(reshape(du, 1, :), vec(DiffResults.value(res))',
-                    DiffResults.jacobian(res), 2, false)
+                resid = __similar(du, length(sol.resid))
+                v, J = DI.value_and_jacobian(_f, resid, AutoForwardDiff(), u)
+                mul!(reshape(du, 1, :), vec(v)', J, 2, false)
                 return nothing
             end
         else
             # For small problems, nesting ForwardDiff is actually quite fast
+            _f = Base.Fix2(prob.f, newprob.p)
             if __is_extension_loaded(Val(:Zygote)) && (length(uu) + length(sol.resid) ≥ 50)
-                _F = @closure (u, p) -> __zygote_compute_nlls_vjp(prob.f, u, p)
+                # TODO: Remove once DI has the value_and_pullback_split defined
+                _F = @closure (u, p) -> __zygote_compute_nlls_vjp(_f, u, p)
             else
                 _F = @closure (u, p) -> begin
-                    T = promote_type(eltype(u), eltype(p))
-                    res = DiffResults.DiffResult(similar(u, T, size(sol.resid)),
-                        similar(u, T, length(sol.resid), length(u)))
-                    ForwardDiff.jacobian!(res, Base.Fix2(prob.f, p), u)
-                    return reshape(
-                        2 .* vec(DiffResults.value(res))' * DiffResults.jacobian(res),
-                        size(u))
+                    _f = Base.Fix2(prob.f, p)
+                    v, J = DI.value_and_jacobian(_f, AutoForwardDiff(), u)
+                    return reshape(2 .* vec(v)' * J, size(u))
                 end
             end
         end
     end
 
-    f_p = __nlsolve_∂f_∂p(prob, _F, uu, p)
-    f_x = __nlsolve_∂f_∂u(prob, _F, uu, p)
+    f_p = __nlsolve_∂f_∂p(prob, _F, uu, newprob.p)
+    f_x = __nlsolve_∂f_∂u(prob, _F, uu, newprob.p)
 
     z_arr = -f_x \ f_p
 
@@ -152,7 +132,7 @@ function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs..
     sumfun = ((z, p),) -> map(zᵢ -> zᵢ * ForwardDiff.partials(p), z)
     if uu isa Number
         partials = sum(sumfun, zip(z_arr, pp))
-    elseif p isa Number
+    elseif pp isa Number
         partials = sumfun((z_arr, pp))
     else
         partials = sum(sumfun, zip(eachcol(z_arr), pp))
@@ -164,7 +144,7 @@ end
 @inline function __nlsolve_∂f_∂p(prob, f::F, u, p) where {F}
     if isinplace(prob)
         __f = p -> begin
-            du = similar(u, promote_type(eltype(u), eltype(p)))
+            du = __similar(u, promote_type(eltype(u), eltype(p)))
             f(du, u, p)
             return du
         end
@@ -182,16 +162,12 @@ end
 
 @inline function __nlsolve_∂f_∂u(prob, f::F, u, p) where {F}
     if isinplace(prob)
-        du = similar(u)
-        __f = (du, u) -> f(du, u, p)
-        ForwardDiff.jacobian(__f, du, u)
+        __f = @closure (du, u) -> f(du, u, p)
+        return ForwardDiff.jacobian(__f, __similar(u), u)
     else
         __f = Base.Fix2(f, p)
-        if u isa Number
-            return ForwardDiff.derivative(__f, u)
-        else
-            return ForwardDiff.jacobian(__f, u)
-        end
+        u isa Number && return ForwardDiff.derivative(__f, u)
+        return ForwardDiff.jacobian(__f, u)
     end
 end
 
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index 5ff5eea40..d3fe1cdd1 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -39,9 +39,9 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
     @bb xo = copy(x)
 
     if setindex_trait(x) === CanSetindex()
-        A = similar(x, length(x), length(x))
-        Aaᵢ = similar(x, length(x))
-        cᵢ = similar(x)
+        A = __similar(x, length(x), length(x))
+        Aaᵢ = __similar(x, length(x))
+        cᵢ = __similar(x)
     else
         A = x
         Aaᵢ = x
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index b34d4cddf..1eeda4fa6 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -272,7 +272,7 @@ end
     return :(return SVector{$N, $T}(($(getcalls...))))
 end
 
-__lbroyden_threshold_cache(x, ::Val{threshold}) where {threshold} = similar(x, threshold)
+__lbroyden_threshold_cache(x, ::Val{threshold}) where {threshold} = __similar(x, threshold)
 function __lbroyden_threshold_cache(x::StaticArray, ::Val{threshold}) where {threshold}
     return zeros(MArray{Tuple{threshold}, eltype(x)})
 end
@@ -298,7 +298,7 @@ end
     end
 end
 function __init_low_rank_jacobian(u, fu, ::Val{threshold}) where {threshold}
-    Vᵀ = similar(u, threshold, length(u))
-    U = similar(u, length(fu), threshold)
+    Vᵀ = __similar(u, threshold, length(u))
+    U = __similar(u, length(fu), threshold)
     return U, Vᵀ
 end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index ae380d7be..0bf027f6f 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -21,13 +21,13 @@ Return the maximum of `a` and `b` if `x1 > x0`, otherwise return the minimum.
 """
 __max_tdir(a, b, x0, x1) = ifelse(x1 > x0, max(a, b), min(a, b))
 
-function __fixed_parameter_function(prob::NonlinearProblem)
+function __fixed_parameter_function(prob::AbstractNonlinearProblem)
     isinplace(prob) && return @closure (du, u) -> prob.f(du, u, prob.p)
     return Base.Fix2(prob.f, prob.p)
 end
 
 function value_and_jacobian(
-        ad, prob::NonlinearProblem, f::F, y, x, cache; J = nothing) where {F}
+        ad, prob::AbstractNonlinearProblem, f::F, y, x, cache; J = nothing) where {F}
     x isa Number && return DI.value_and_derivative(f, ad, x, cache)
 
     if isinplace(prob)
@@ -46,21 +46,22 @@ function value_and_jacobian(
     end
 end
 
-function jacobian_cache(ad, prob::NonlinearProblem, f::F, y, x) where {F}
+function jacobian_cache(ad, prob::AbstractNonlinearProblem, f::F, y, x) where {F}
     x isa Number && return (nothing, DI.prepare_derivative(f, ad, x))
 
     if isinplace(prob)
-        J = similar(y, length(y), length(x))
+        J = __similar(y, length(y), length(x))
         SciMLBase.has_jac(prob.f) && return J, HasAnalyticJacobian()
         return J, DI.prepare_jacobian(f, y, ad, x)
     else
         SciMLBase.has_jac(prob.f) && return nothing, HasAnalyticJacobian()
-        J = ArrayInterface.can_setindex(x) ? similar(y, length(y), length(x)) : nothing
+        J = ArrayInterface.can_setindex(x) ? __similar(y, length(y), length(x)) : nothing
         return J, DI.prepare_jacobian(f, ad, x)
     end
 end
 
-function compute_jacobian_and_hessian(ad, prob::NonlinearProblem, f::F, y, x) where {F}
+function compute_jacobian_and_hessian(
+        ad, prob::AbstractNonlinearProblem, f::F, y, x) where {F}
     if x isa Number
         df = @closure x -> DI.derivative(f, ad, x)
         return f(x), df(x), DI.derivative(df, ad, x)
@@ -68,7 +69,7 @@ function compute_jacobian_and_hessian(ad, prob::NonlinearProblem, f::F, y, x) wh
 
     if isinplace(prob)
         df = @closure x -> begin
-            res = similar(y, promote_type(eltype(y), eltype(x)))
+            res = __similar(y, promote_type(eltype(y), eltype(x)))
             return DI.jacobian(f, res, ad, x)
         end
         J, H = DI.value_and_jacobian(df, ad, x)
@@ -83,7 +84,7 @@ end
 __init_identity_jacobian(u::Number, fu, α = true) = oftype(u, α)
 __init_identity_jacobian!!(J::Number) = one(J)
 function __init_identity_jacobian(u, fu, α = true)
-    J = similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u))
+    J = __similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u))
     fill!(J, zero(eltype(J)))
     J[diagind(J)] .= eltype(J)(α)
     return J
@@ -129,7 +130,7 @@ end
             T = eltype(x)
             return T.(f.resid_prototype)
         else
-            fx = similar(x)
+            fx = __similar(x)
             f(fx, x, p)
             return fx
         end
@@ -242,3 +243,16 @@ end
 
 # Extension
 function __zygote_compute_nlls_vjp end
+
+function __similar(x, args...; kwargs...)
+    y = similar(x, args...; kwargs...)
+    return __init_bigfloat_array!!(y)
+end
+
+function __init_bigfloat_array!!(x)
+    if ArrayInterface.can_setindex(x)
+        eltype(x) <: BigFloat && fill!(x, BigFloat(0))
+        return x
+    end
+    return x
+end

From 86917a522581ae49d12f0273b100703452b7c25e Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 18:29:32 -0700
Subject: [PATCH 340/375] Remove the static arrays special casing

---
 lib/SimpleNonlinearSolve/Project.toml             | 10 +++++-----
 .../ext/SimpleNonlinearSolveStaticArraysExt.jl    |  7 -------
 .../src/SimpleNonlinearSolve.jl                   |  1 +
 lib/SimpleNonlinearSolve/src/ad.jl                |  6 ++++--
 lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl    | 15 +++++++++------
 lib/SimpleNonlinearSolve/src/utils.jl             | 14 +++++++-------
 .../test/core/rootfind_tests.jl                   |  9 ++++-----
 7 files changed, 30 insertions(+), 32 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index a7aa31f82..1d9cc13ab 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -18,19 +18,18 @@ MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 
 [weakdeps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [extensions]
 SimpleNonlinearSolveChainRulesCoreExt = "ChainRulesCore"
 SimpleNonlinearSolveReverseDiffExt = "ReverseDiff"
-SimpleNonlinearSolveStaticArraysExt = "StaticArrays"
 SimpleNonlinearSolveTrackerExt = "Tracker"
 SimpleNonlinearSolveZygoteExt = "Zygote"
 
@@ -40,7 +39,7 @@ AllocCheck = "0.1.1"
 Aqua = "0.8"
 ArrayInterface = "7.9"
 CUDA = "5.2"
-ChainRulesCore = "1.22"
+ChainRulesCore = "1.23"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.149"
 DiffResults = "1.1"
@@ -59,13 +58,14 @@ PrecompileTools = "1.2"
 Random = "1.10"
 ReTestItems = "1.23"
 Reexport = "1.2"
-ReverseDiff = "1.15"
+ReverseDiff = "1.15.3"
 SciMLBase = "2.37.0"
 SciMLSensitivity = "7.58"
+Setfield = "1.1.1"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"
 Test = "1.10"
-Tracker = "0.2.32"
+Tracker = "0.2.33"
 Zygote = "0.6.69"
 julia = "1.10"
 
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl
deleted file mode 100644
index c865084ce..000000000
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveStaticArraysExt.jl
+++ /dev/null
@@ -1,7 +0,0 @@
-module SimpleNonlinearSolveStaticArraysExt
-
-using SimpleNonlinearSolve: SimpleNonlinearSolve
-
-@inline SimpleNonlinearSolve.__is_extension_loaded(::Val{:StaticArrays}) = true
-
-end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 04a32c969..eba6d991e 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -24,6 +24,7 @@ using PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidati
                      NonlinearFunction, NonlinearLeastSquaresProblem, NonlinearProblem,
                      ReturnCode, init, remake, solve, AbstractNonlinearAlgorithm,
                      build_solution, isinplace, _unwrap_val
+    using Setfield: @set!
     using StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
 end
 
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index ffae7cc27..bb5afea8f 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -109,10 +109,12 @@ function __nlsolve_ad(prob::NonlinearLeastSquaresProblem, alg, args...; kwargs..
             end
         else
             # For small problems, nesting ForwardDiff is actually quite fast
-            _f = Base.Fix2(prob.f, newprob.p)
             if __is_extension_loaded(Val(:Zygote)) && (length(uu) + length(sol.resid) ≥ 50)
                 # TODO: Remove once DI has the value_and_pullback_split defined
-                _F = @closure (u, p) -> __zygote_compute_nlls_vjp(_f, u, p)
+                _F = @closure (u, p) -> begin
+                    _f = Base.Fix2(prob.f, p)
+                    return __zygote_compute_nlls_vjp(_f, u, p)
+                end
             else
                 _F = @closure (u, p) -> begin
                     _f = Base.Fix2(prob.f, p)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 7dd152277..835ee4b50 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -77,12 +77,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...
     α_1 = one(T)
     f_1 = fx_norm
 
-    history_f_k = if x isa SArray ||
-                     (x isa Number && __is_extension_loaded(Val(:StaticArrays)))
-        ones(SVector{M, T}) * fx_norm
-    else
-        fill(fx_norm, M)
-    end
+    history_f_k = x isa SArray ? ones(SVector{M, T}) * fx_norm :
+                  __history_vec(fx_norm, Val(M))
 
     # Generate the cache
     @bb x_cache = similar(x)
@@ -150,6 +146,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...
         # Store function value
         if history_f_k isa SVector
             history_f_k = Base.setindex(history_f_k, fx_norm_new, mod1(k, M))
+        elseif history_f_k isa NTuple
+            @set! history_f_k[mod1(k, M)] = fx_norm_new
         else
             history_f_k[mod1(k, M)] = fx_norm_new
         end
@@ -158,3 +156,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...
 
     return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
 end
+
+@inline @generated function __history_vec(fx_norm, ::Val{M}) where {M}
+    M ≥ 11 && return :(fill(fx_norm, M)) # Julia can't specialize here
+    return :(ntuple(Returns(fx_norm), $(M)))
+end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 0bf027f6f..2e76a4b6e 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -32,16 +32,15 @@ function value_and_jacobian(
 
     if isinplace(prob)
         if cache isa HasAnalyticJacobian
-            prob.f.jac(J, x, p)
+            prob.f.jac(J, x, prob.p)
             f(y, x)
-        else
-            DI.jacobian!(f, y, J, ad, x, cache)
+            return y, J
         end
-        return y, J
+        return DI.value_and_jacobian!(f, y, J, ad, x, cache)
     else
         cache isa HasAnalyticJacobian && return f(x), prob.f.jac(x, prob.p)
         J === nothing && return DI.value_and_jacobian(f, ad, x, cache)
-        y, _ = DI.value_and_jacobian!(f, J, ad, x, cache)
+        y, J = DI.value_and_jacobian!(f, J, ad, x, cache)
         return y, J
     end
 end
@@ -63,8 +62,9 @@ end
 function compute_jacobian_and_hessian(
         ad, prob::AbstractNonlinearProblem, f::F, y, x) where {F}
     if x isa Number
-        df = @closure x -> DI.derivative(f, ad, x)
-        return f(x), df(x), DI.derivative(df, ad, x)
+        H = DI.second_derivative(f, ad, x)
+        v, J = DI.value_and_derivative(f, ad, x)
+        return v, J, H
     end
 
     if isinplace(prob)
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index 1ef0757b8..6165314b5 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -34,7 +34,6 @@ end
 
 export quadratic_f, quadratic_f!, quadratic_f2, newton_fails, TERMINATION_CONDITIONS,
        benchmark_nlsolve_oop, benchmark_nlsolve_iip
-
 end
 
 @testitem "First Order Methods" setup=[RootfindingTesting] tags=[:core] begin
@@ -42,7 +41,7 @@ end
         SimpleTrustRegion,
         (args...; kwargs...) -> SimpleTrustRegion(
             args...; nlsolve_update_rule = Val(true), kwargs...))
-        @testset "AutoDiff: $(nameof(typeof(autodiff))))" for autodiff in (
+        @testset "AutoDiff: $(nameof(typeof(autodiff)))" for autodiff in (
             AutoFiniteDiff(), AutoForwardDiff(), AutoPolyesterForwardDiff())
             @testset "[OOP] u0: $(typeof(u0))" for u0 in (
                 [1.0, 1.0], @SVector[1.0, 1.0], 1.0)
@@ -59,7 +58,7 @@ end
             end
         end
 
-        @testset "Termination condition: $(termination_condition) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+        @testset "Termination condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
             u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
             probN = NonlinearProblem(quadratic_f, u0, 2.0)
@@ -79,7 +78,7 @@ end
         end
     end
 
-    @testset "Termination condition: $(termination_condition) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+    @testset "Termination condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
         u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
         probN = NonlinearProblem(quadratic_f, u0, 2.0)
@@ -104,7 +103,7 @@ end
             @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
         end
 
-        @testset "Termination condition: $(termination_condition) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+        @testset "Termination condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
             u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
 
             probN = NonlinearProblem(quadratic_f, u0, 2.0)

From 4fd1bceb7b2a9ddedb066212974af4a891c0ba64 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Sat, 25 May 2024 20:07:01 -0700
Subject: [PATCH 341/375] Add Halley to 23 test problems

---
 .../test/core/23_test_problems_tests.jl       | 23 +++++++++++++++----
 .../test/core/rootfind_tests.jl               |  6 +++++
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
index 9625c6872..e45560912 100644
--- a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
@@ -41,7 +41,7 @@ end
 export problems, dicts, test_on_library
 end
 
-@testitem "SimpleNewtonRaphson" setup=[RobustnessTesting] tags=[:core] begin
+@testitem "23 Test Problems: SimpleNewtonRaphson" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleNewtonRaphson(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -50,7 +50,20 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "SimpleTrustRegion" setup=[RobustnessTesting] tags=[:core] begin
+@testitem "23 Test Problems: SimpleHalley" setup=[RobustnessTesting] tags=[:core] begin
+    alg_ops = (SimpleHalley(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    if Sys.isapple()
+        broken_tests[alg_ops[1]] = [1, 5, 11, 15, 16, 18]
+    else
+        broken_tests[alg_ops[1]] = [1, 5, 15, 16, 18]
+    end
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "23 Test Problems: SimpleTrustRegion" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)))
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -60,7 +73,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "SimpleDFSane" setup=[RobustnessTesting] tags=[:core] begin
+@testitem "23 Test Problems: SimpleDFSane" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleDFSane(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -73,7 +86,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "SimpleBroyden" retries=5 setup=[RobustnessTesting] tags=[:core] begin
+@testitem "23 Test Problems: SimpleBroyden" retries=5 setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleBroyden(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -82,7 +95,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "SimpleKlement" setup=[RobustnessTesting] tags=[:core] begin
+@testitem "23 Test Problems: SimpleKlement" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleKlement(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index 6165314b5..eecdeca18 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -76,6 +76,12 @@ end
             @test SciMLBase.successful_retcode(sol)
             @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
         end
+
+        @testset "[IIP] u0: $(nameof(typeof(u0)))" for u0 in ([1.0, 1.0],)
+            sol = benchmark_nlsolve_iip(quadratic_f!, u0; solver = SimpleHalley(; autodiff))
+            @test SciMLBase.successful_retcode(sol)
+            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+        end
     end
 
     @testset "Termination condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,

From d4a6c2a313618f639771f72c2cff1218be52d526 Mon Sep 17 00:00:00 2001
From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com>
Date: Thu, 30 May 2024 08:56:14 +0200
Subject: [PATCH 342/375] Bump DifferentiationInterface compat to 0.5

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 1d9cc13ab..d2915a847 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -43,7 +43,7 @@ ChainRulesCore = "1.23"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.149"
 DiffResults = "1.1"
-DifferentiationInterface = "0.4"
+DifferentiationInterface = "0.5"
 ExplicitImports = "1.5.0"
 FastClosures = "0.3.2"
 FiniteDiff = "2.23.1"

From fdd16a7728614c04a156cc909028b1b8c9c892c1 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Fri, 7 Jun 2024 08:12:07 -0400
Subject: [PATCH 343/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index d2915a847..c53c17b93 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.9.0"
+version = "1.10.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From 9702729b21c8f24b536dfdd5079855609e2f7ae9 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 27 Jun 2024 10:10:20 +0800
Subject: [PATCH 344/375] Remove recompile_invalidations

`@recompile_invalidations` should only be used in very specific scenarios, and this is not one of those scenarios. Also, there are big changes being done with https://github.com/SciML/CommonWorldInvalidations.jl. With that, we only need to `@recompile_invalidations` on a few entry points. In particular, Static.jl, Symbolics.jl, and preferably ForwardDiff.jl and StaticArrays.jl would adopt it too. But this means that in order to handle all of this effectively, in SciML we only need to apply it on Static.jl, Symbolics.jl, and SciMLBase.jl and the whole ecosystem should be fine.

In any case, this library doesn't need it. It shouldn't make a tangible difference in compile times, while it increases precompile times by a lot.
---
 .../src/SimpleNonlinearSolve.jl               | 52 +++++++++----------
 1 file changed, 25 insertions(+), 27 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index eba6d991e..ef3947f4a 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -1,32 +1,30 @@
 module SimpleNonlinearSolve
 
-using PrecompileTools: @compile_workload, @setup_workload, @recompile_invalidations
-
-@recompile_invalidations begin
-    using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff,
-                   AutoPolyesterForwardDiff
-    using ArrayInterface: ArrayInterface
-    using ConcreteStructs: @concrete
-    using DiffEqBase: DiffEqBase, AbstractNonlinearTerminationMode,
-                      AbstractSafeNonlinearTerminationMode,
-                      AbstractSafeBestNonlinearTerminationMode, AbsNormTerminationMode,
-                      NONLINEARSOLVE_DEFAULT_NORM
-    using DifferentiationInterface: DifferentiationInterface
-    using DiffResults: DiffResults
-    using FastClosures: @closure
-    using FiniteDiff: FiniteDiff
-    using ForwardDiff: ForwardDiff, Dual
-    using LinearAlgebra: LinearAlgebra, I, convert, copyto!, diagind, dot, issuccess, lu,
-                         mul!, norm, transpose
-    using MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
-    using Reexport: @reexport
-    using SciMLBase: SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem,
-                     NonlinearFunction, NonlinearLeastSquaresProblem, NonlinearProblem,
-                     ReturnCode, init, remake, solve, AbstractNonlinearAlgorithm,
-                     build_solution, isinplace, _unwrap_val
-    using Setfield: @set!
-    using StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
-end
+using PrecompileTools: @compile_workload, @setup_workload
+
+using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff,
+               AutoPolyesterForwardDiff
+using ArrayInterface: ArrayInterface
+using ConcreteStructs: @concrete
+using DiffEqBase: DiffEqBase, AbstractNonlinearTerminationMode,
+                  AbstractSafeNonlinearTerminationMode,
+                  AbstractSafeBestNonlinearTerminationMode, AbsNormTerminationMode,
+                  NONLINEARSOLVE_DEFAULT_NORM
+using DifferentiationInterface: DifferentiationInterface
+using DiffResults: DiffResults
+using FastClosures: @closure
+using FiniteDiff: FiniteDiff
+using ForwardDiff: ForwardDiff, Dual
+using LinearAlgebra: LinearAlgebra, I, convert, copyto!, diagind, dot, issuccess, lu,
+                     mul!, norm, transpose
+using MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
+using Reexport: @reexport
+using SciMLBase: SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem,
+                 NonlinearFunction, NonlinearLeastSquaresProblem, NonlinearProblem,
+                 ReturnCode, init, remake, solve, AbstractNonlinearAlgorithm,
+                 build_solution, isinplace, _unwrap_val
+using Setfield: @set!
+using StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
 
 const DI = DifferentiationInterface
 

From 9ea50fcabfed4ffc998fb222bb971a7d05451536 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 27 Jun 2024 10:22:11 +0800
Subject: [PATCH 345/375] Update src/SimpleNonlinearSolve.jl

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index ef3947f4a..b42809c52 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -15,8 +15,8 @@ using DiffResults: DiffResults
 using FastClosures: @closure
 using FiniteDiff: FiniteDiff
 using ForwardDiff: ForwardDiff, Dual
-using LinearAlgebra: LinearAlgebra, I, convert, copyto!, diagind, dot, issuccess, lu,
-                     mul!, norm, transpose
+using LinearAlgebra: LinearAlgebra, I, convert, copyto!, diagind, dot, issuccess, lu, mul!,
+                     norm, transpose
 using MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
 using Reexport: @reexport
 using SciMLBase: SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem,

From b186343d5496e1e64912ef3616b5e4f7ae3544e1 Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Thu, 27 Jun 2024 10:23:31 +0800
Subject: [PATCH 346/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index c53c17b93..c27d74903 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.10.0"
+version = "1.10.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From 86da8b3b41af1985221ef697b4930860ace371bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez?= <daniel.glez.arribas@gmail.com>
Date: Sat, 13 Jul 2024 14:16:02 +0200
Subject: [PATCH 347/375] New default values

---
 .../src/bracketing/itp.jl                     | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index 2972d1c5c..4a542a958 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -17,13 +17,13 @@ The following keyword parameters are accepted.
   - `n₀::Int = 10`, the 'slack'. Must not be negative. When n₀ = 0 the worst-case is
     identical to that of bisection, but increacing n₀ provides greater oppotunity for
     superlinearity.
-  - `κ₁::Float64 = 0.007`. Must not be negative. The recomended value is `0.2/(x₂ - x₁)`.
+  - `scaled_κ₁::Float64 = 0.2`. Must not be negative. The recomended value is `0.2`.
     Lower values produce tighter asymptotic behaviour, while higher values improve the
     steady-state behaviour when truncation is not helpful.
-  - `κ₂::Real = 1.5`. Must lie in [1, 1+ϕ ≈ 2.62). Higher values allow for a greater
+  - `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62). Higher values allow for a greater
     convergence rate, but also make the method more succeptable to worst-case performance.
-    In practice, κ=1,2 seems to work well due to the computational simplicity, as κ₂ is used
-    as an exponent in the method.
+    In practice, κ=1, 2 seems to work well due to the computational simplicity, as κ₂ is
+    used as an exponent in the method.
 
 ### Worst Case Performance
 
@@ -35,19 +35,19 @@ n½ + `n₀` iterations, where n½ is the number of iterations using bisection
 If `f` is twice differentiable and the root is simple, then with `n₀` > 0 the convergence
 rate is √`κ₂`.
 """
-struct ITP{T} <: AbstractBracketingAlgorithm
-    k1::T
-    k2::T
+struct ITP{T₁, T₂} <: AbstractBracketingAlgorithm
+    scaled_k1::T₁
+    k2::T₂
     n0::Int
-    function ITP(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
-        k1 < 0 && error("Hyper-parameter κ₁ should not be negative")
+    function ITP(;
+            scaled_k1::T₁ = 0.2, k2::T₂ = 2, n0::Int = 10) where {T₁ <: Real, T₂ <: Real}
+        scaled_k1 < 0 && error("Hyper-parameter κ₁ should not be negative")
         n0 < 0 && error("Hyper-parameter n₀ should not be negative")
         if k2 < 1 || k2 > (1.5 + sqrt(5) / 2)
             throw(ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where \
                                  ϕ ≈ 1.618... is the golden ratio"))
         end
-        T = promote_type(eltype(k1), eltype(k2))
-        return new{T}(k1, k2, n0)
+        return new{T₁, T₂}(scaled_k1, k2, n0)
     end
 end
 
@@ -72,7 +72,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
     end
     ϵ = abstol
     #defining variables/cache
-    k1 = alg.k1
+    k1 = alg.scaled_k1 / abs(right - left)
     k2 = alg.k2
     n0 = alg.n0
     n_h = ceil(log2(abs(right - left) / (2 * ϵ)))
@@ -88,7 +88,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
     while i <= maxiters
         span = abs(right - left)
         r = ϵ_s - (span / 2)
-        δ = k1 * (span^k2)
+        δ = k1 * ((k2 == 2) ? span^2 : (span^k2))
 
         ## Interpolation step ##
         x_f = left + (right - left) * (fl / (fl - fr))

From d309aea0ec6f09824f0c0c5b3cc425cc94ee418c Mon Sep 17 00:00:00 2001
From: Matt Bossart <bossart.matt@gmail.com>
Date: Tue, 16 Jul 2024 17:11:56 -0400
Subject: [PATCH 348/375] add ImmutableNonlinearProblem

---
 .../SimpleNonlinearSolveChainRulesCoreExt.jl  |  6 +-
 .../ext/SimpleNonlinearSolveReverseDiffExt.jl |  6 +-
 .../ext/SimpleNonlinearSolveTrackerExt.jl     |  6 +-
 .../src/SimpleNonlinearSolve.jl               | 21 ++++--
 lib/SimpleNonlinearSolve/src/ad.jl            |  4 +-
 .../src/immutable_nonlinear_problem.jl        | 67 +++++++++++++++++++
 .../src/nlsolve/broyden.jl                    |  2 +-
 .../src/nlsolve/dfsane.jl                     |  2 +-
 .../src/nlsolve/halley.jl                     |  2 +-
 .../src/nlsolve/klement.jl                    |  2 +-
 .../src/nlsolve/lbroyden.jl                   |  6 +-
 .../src/nlsolve/raphson.jl                    |  2 +-
 .../src/nlsolve/trustRegion.jl                |  2 +-
 lib/SimpleNonlinearSolve/src/utils.jl         |  8 +--
 .../test/core/adjoint_tests.jl                |  6 +-
 .../test/gpu/cuda_tests.jl                    |  1 +
 16 files changed, 112 insertions(+), 31 deletions(-)
 create mode 100644 lib/SimpleNonlinearSolve/src/immutable_nonlinear_problem.jl

diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
index 2987f1c5b..240b127c7 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
@@ -2,13 +2,13 @@ module SimpleNonlinearSolveChainRulesCoreExt
 
 using ChainRulesCore: ChainRulesCore, NoTangent
 using DiffEqBase: DiffEqBase
-using SciMLBase: ChainRulesOriginator, NonlinearProblem, NonlinearLeastSquaresProblem
-using SimpleNonlinearSolve: SimpleNonlinearSolve
+using SciMLBase: ChainRulesOriginator, NonlinearLeastSquaresProblem
+using SimpleNonlinearSolve: SimpleNonlinearSolve, ImmutableNonlinearProblem
 
 # The expectation here is that no-one is using this directly inside a GPU kernel. We can
 # eventually lift this requirement using a custom adjoint
 function ChainRulesCore.rrule(::typeof(SimpleNonlinearSolve.__internal_solve_up),
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+        prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
         sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...)
     out, ∇internal = DiffEqBase._solve_adjoint(
         prob, sensealg, u0, p, ChainRulesOriginator(), alg, args...; kwargs...)
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
index 249bbbedb..6cf6f4d3c 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
@@ -3,11 +3,11 @@ module SimpleNonlinearSolveReverseDiffExt
 using ArrayInterface: ArrayInterface
 using DiffEqBase: DiffEqBase
 using ReverseDiff: ReverseDiff, TrackedArray, TrackedReal
-using SciMLBase: ReverseDiffOriginator, NonlinearProblem, NonlinearLeastSquaresProblem
-using SimpleNonlinearSolve: SimpleNonlinearSolve
+using SciMLBase: ReverseDiffOriginator, NonlinearLeastSquaresProblem
+using SimpleNonlinearSolve: SimpleNonlinearSolve, ImmutableNonlinearProblem
 import SimpleNonlinearSolve: __internal_solve_up
 
-for pType in (NonlinearProblem, NonlinearLeastSquaresProblem)
+for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem)
     @eval begin
         function __internal_solve_up(prob::$(pType), sensealg, u0::TrackedArray, u0_changed,
                 p::TrackedArray, p_changed, alg, args...; kwargs...)
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
index a212b220e..679274754 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
@@ -1,11 +1,11 @@
 module SimpleNonlinearSolveTrackerExt
 
 using DiffEqBase: DiffEqBase
-using SciMLBase: TrackerOriginator, NonlinearProblem, NonlinearLeastSquaresProblem, remake
-using SimpleNonlinearSolve: SimpleNonlinearSolve
+using SciMLBase: TrackerOriginator, NonlinearLeastSquaresProblem, remake
+using SimpleNonlinearSolve: SimpleNonlinearSolve, ImmutableNonlinearProblem
 using Tracker: Tracker, TrackedArray
 
-for pType in (NonlinearProblem, NonlinearLeastSquaresProblem)
+for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem)
     @eval begin
         function SimpleNonlinearSolve.__internal_solve_up(
                 prob::$(pType), sensealg, u0::TrackedArray,
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index b42809c52..0baa27655 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -19,10 +19,11 @@ using LinearAlgebra: LinearAlgebra, I, convert, copyto!, diagind, dot, issuccess
                      norm, transpose
 using MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
 using Reexport: @reexport
-using SciMLBase: SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem,
+using SciMLBase: @add_kwonly, SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem,
+                 AbstractNonlinearFunction, StandardNonlinearProblem,
                  NonlinearFunction, NonlinearLeastSquaresProblem, NonlinearProblem,
                  ReturnCode, init, remake, solve, AbstractNonlinearAlgorithm,
-                 build_solution, isinplace, _unwrap_val
+                 build_solution, isinplace, _unwrap_val, warn_paramtype
 using Setfield: @set!
 using StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
 
@@ -35,7 +36,7 @@ abstract type AbstractBracketingAlgorithm <: AbstractSimpleNonlinearSolveAlgorit
 abstract type AbstractNewtonAlgorithm <: AbstractSimpleNonlinearSolveAlgorithm end
 
 @inline __is_extension_loaded(::Val) = false
-
+include("immutable_nonlinear_problem.jl")
 include("utils.jl")
 include("linesearch.jl")
 
@@ -70,6 +71,18 @@ end
 # By Pass the highlevel checks for NonlinearProblem for Simple Algorithms
 function SciMLBase.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
         args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...)
+    prob = convert(ImmutableNonlinearProblem, prob)
+    if sensealg === nothing && haskey(prob.kwargs, :sensealg)
+        sensealg = prob.kwargs[:sensealg]
+    end
+    new_u0 = u0 !== nothing ? u0 : prob.u0
+    new_p = p !== nothing ? p : prob.p
+    return __internal_solve_up(prob, sensealg, new_u0, u0 === nothing, new_p,
+        p === nothing, alg, args...; prob.kwargs..., kwargs...)
+end
+
+function SciMLBase.solve(prob::ImmutableNonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
+    args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...)
     if sensealg === nothing && haskey(prob.kwargs, :sensealg)
         sensealg = prob.kwargs[:sensealg]
     end
@@ -79,7 +92,7 @@ function SciMLBase.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSol
         p === nothing, alg, args...; prob.kwargs..., kwargs...)
 end
 
-function __internal_solve_up(_prob::NonlinearProblem, sensealg, u0, u0_changed,
+function __internal_solve_up(_prob::ImmutableNonlinearProblem, sensealg, u0, u0_changed,
         p, p_changed, alg, args...; kwargs...)
     prob = u0_changed || p_changed ? remake(_prob; u0, p) : _prob
     return SciMLBase.__solve(prob, alg, args...; kwargs...)
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index bb5afea8f..353cc0b9d 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -1,4 +1,4 @@
-for pType in (NonlinearProblem, NonlinearLeastSquaresProblem)
+for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem)
     @eval function SciMLBase.solve(
             prob::$(pType){<:Union{Number, <:AbstractArray}, iip,
                 <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
@@ -31,7 +31,7 @@ for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
 end
 
 function __nlsolve_ad(
-        prob::Union{IntervalNonlinearProblem, NonlinearProblem}, alg, args...; kwargs...)
+        prob::Union{IntervalNonlinearProblem, ImmutableNonlinearProblem}, alg, args...; kwargs...)
     p = value(prob.p)
     if prob isa IntervalNonlinearProblem
         tspan = value.(prob.tspan)
diff --git a/lib/SimpleNonlinearSolve/src/immutable_nonlinear_problem.jl b/lib/SimpleNonlinearSolve/src/immutable_nonlinear_problem.jl
new file mode 100644
index 000000000..ca1dfc079
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/immutable_nonlinear_problem.jl
@@ -0,0 +1,67 @@
+struct ImmutableNonlinearProblem{uType, isinplace, P, F, K, PT} <:
+    AbstractNonlinearProblem{uType, isinplace}
+ f::F
+ u0::uType
+ p::P
+ problem_type::PT
+ kwargs::K
+ @add_kwonly function ImmutableNonlinearProblem{iip}(f::AbstractNonlinearFunction{iip}, u0,
+         p = NullParameters(),
+         problem_type = StandardNonlinearProblem();
+         kwargs...) where {iip}
+     if haskey(kwargs, :p)
+         error("`p` specified as a keyword argument `p = $(kwargs[:p])` to `NonlinearProblem`. This is not supported.")
+     end
+     warn_paramtype(p)
+     new{typeof(u0), iip, typeof(p), typeof(f),
+         typeof(kwargs), typeof(problem_type)}(f,
+         u0,
+         p,
+         problem_type,
+         kwargs)
+ end
+
+ """
+ Define a steady state problem using the given function.
+ `isinplace` optionally sets whether the function is inplace or not.
+ This is determined automatically, but not inferred.
+ """
+ function ImmutableNonlinearProblem{iip}(f, u0, p = NullParameters(); kwargs...) where {iip}
+     ImmutableNonlinearProblem{iip}(NonlinearFunction{iip}(f), u0, p; kwargs...)
+ end
+end
+
+"""
+Define a nonlinear problem using an instance of
+[`AbstractNonlinearFunction`](@ref AbstractNonlinearFunction).
+"""
+function ImmutableNonlinearProblem(f::AbstractNonlinearFunction, u0, p = NullParameters(); kwargs...)
+ ImmutableNonlinearProblem{isinplace(f)}(f, u0, p; kwargs...)
+end
+
+function ImmutableNonlinearProblem(f, u0, p = NullParameters(); kwargs...)
+ ImmutableNonlinearProblem(NonlinearFunction(f), u0, p; kwargs...)
+end
+
+"""
+Define a ImmutableNonlinearProblem problem from SteadyStateProblem
+"""
+function ImmutableNonlinearProblem(prob::AbstractNonlinearProblem)
+ ImmutableNonlinearProblem{isinplace(prob)}(prob.f, prob.u0, prob.p)
+end
+
+
+function Base.convert(::Type{ImmutableNonlinearProblem}, prob::T) where {T <: NonlinearProblem}
+ ImmutableNonlinearProblem{isinplace(prob)}(prob.f,
+     prob.u0,
+     prob.p,
+     prob.problem_type;
+     prob.kwargs...)
+end
+
+function DiffEqBase.get_concrete_problem(prob::ImmutableNonlinearProblem, isadapt; kwargs...)
+    u0 = DiffEqBase.get_concrete_u0(prob, isadapt, nothing, kwargs)
+    u0 = DiffEqBase.promote_u0(u0, prob.p, nothing)
+    p = DiffEqBase.get_concrete_p(prob, kwargs)
+    DiffEqBase.remake(prob; u0 = u0, p = p)
+end
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
index 890578f5d..2b02955d6 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/broyden.jl
@@ -22,7 +22,7 @@ end
 
 __get_linesearch(::SimpleBroyden{LS}) where {LS} = Val(LS)
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleBroyden, args...;
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleBroyden, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
index 835ee4b50..746066605 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/dfsane.jl
@@ -54,7 +54,7 @@ function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real =
         σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp, η_strategy)
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleDFSane{M}, args...;
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleDFSane{M}, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
         termination_condition = nothing, kwargs...) where {M}
     x = __maybe_unaliased(prob.u0, alias_u0)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
index d3fe1cdd1..bf2f5f8b5 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/halley.jl
@@ -24,7 +24,7 @@ A low-overhead implementation of Halley's Method.
     autodiff = nothing
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleHalley, args...;
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleHalley, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
index 8041ef4b8..2db34059b 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/klement.jl
@@ -6,7 +6,7 @@ method is non-allocating on scalar and static array problems.
 """
 struct SimpleKlement <: AbstractSimpleNonlinearSolveAlgorithm end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleKlement, args...;
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleKlement, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 1eeda4fa6..683e9e592 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -29,7 +29,7 @@ function SimpleLimitedMemoryBroyden(;
     return SimpleLimitedMemoryBroyden{_unwrap_val(threshold), _unwrap_val(linesearch)}(alpha)
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleLimitedMemoryBroyden,
         args...; termination_condition = nothing, kwargs...)
     if prob.u0 isa SArray
         if termination_condition === nothing ||
@@ -44,7 +44,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyd
     return __generic_solve(prob, alg, args...; termination_condition, kwargs...)
 end
 
-@views function __generic_solve(prob::NonlinearProblem, alg::SimpleLimitedMemoryBroyden,
+@views function __generic_solve(prob::ImmutableNonlinearProblem, alg::SimpleLimitedMemoryBroyden,
         args...; abstol = nothing, reltol = nothing, maxiters = 1000,
         alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
@@ -114,7 +114,7 @@ end
 # Non-allocating StaticArrays version of SimpleLimitedMemoryBroyden is actually quite
 # finicky, so we'll implement it separately from the generic version
 # Ignore termination_condition. Don't pass things into internal functions
-function __static_solve(prob::NonlinearProblem{<:SArray}, alg::SimpleLimitedMemoryBroyden,
+function __static_solve(prob::ImmutableNonlinearProblem{<:SArray}, alg::SimpleLimitedMemoryBroyden,
         args...; abstol = nothing, maxiters = 1000, kwargs...)
     x = prob.u0
     fx = _get_fx(prob, x)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
index ca6864b28..d7e72dbdf 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
@@ -23,7 +23,7 @@ end
 
 const SimpleGaussNewton = SimpleNewtonRaphson
 
-function SciMLBase.__solve(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+function SciMLBase.__solve(prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
         alg::SimpleNewtonRaphson, args...; abstol = nothing, reltol = nothing,
         maxiters = 1000, termination_condition = nothing, alias_u0 = false, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 6ff263ecd..2e78baf02 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -55,7 +55,7 @@ scalar and static array problems.
     nlsolve_update_rule = Val(false)
 end
 
-function SciMLBase.__solve(prob::NonlinearProblem, alg::SimpleTrustRegion, args...;
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleTrustRegion, args...;
         abstol = nothing, reltol = nothing, maxiters = 1000,
         alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 2e76a4b6e..9380ff186 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -123,7 +123,7 @@ end
         error("Inplace NonlinearLeastSquaresProblem requires a `resid_prototype`")
     return _get_fx(prob.f, x, prob.p)
 end
-@inline _get_fx(prob::NonlinearProblem, x) = _get_fx(prob.f, x, prob.p)
+@inline _get_fx(prob::ImmutableNonlinearProblem, x) = _get_fx(prob.f, x, prob.p)
 @inline function _get_fx(f::NonlinearFunction, x, p)
     if isinplace(f)
         if f.resid_prototype !== nothing
@@ -145,7 +145,7 @@ end
 # different. NonlinearSolve is more for robust / cached solvers while SimpleNonlinearSolve
 # is meant for low overhead solvers, users can opt into the other termination modes but the
 # default is to use the least overhead version.
-function init_termination_cache(prob::NonlinearProblem, abstol, reltol, du, u, ::Nothing)
+function init_termination_cache(prob::ImmutableNonlinearProblem, abstol, reltol, du, u, ::Nothing)
     return init_termination_cache(
         prob, abstol, reltol, du, u, AbsNormTerminationMode(Base.Fix1(maximum, abs)))
 end
@@ -155,14 +155,14 @@ function init_termination_cache(
         prob, abstol, reltol, du, u, AbsNormTerminationMode(Base.Fix2(norm, 2)))
 end
 
-function init_termination_cache(prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
+function init_termination_cache(prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
         abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode)
     T = promote_type(eltype(du), eltype(u))
     abstol = __get_tolerance(u, abstol, T)
     reltol = __get_tolerance(u, reltol, T)
     tc_ = if hasfield(typeof(tc), :internalnorm) && tc.internalnorm === nothing
         internalnorm = ifelse(
-            prob isa NonlinearProblem, Base.Fix1(maximum, abs), Base.Fix2(norm, 2))
+            prob isa ImmutableNonlinearProblem, Base.Fix1(maximum, abs), Base.Fix2(norm, 2))
         DiffEqBase.set_termination_mode_internalnorm(tc, internalnorm)
     else
         tc
diff --git a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
index 749be1ada..a4757f10d 100644
--- a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
@@ -13,9 +13,9 @@
     p = [3.0, 2.0]
 
     ∂p_zygote = only(Zygote.gradient(solve_nlprob, p))
-    ∂p_forwarddiff = ForwardDiff.gradient(solve_nlprob, p)
+    #∂p_forwarddiff = ForwardDiff.gradient(solve_nlprob, p)
     ∂p_tracker = Tracker.data(only(Tracker.gradient(solve_nlprob, p)))
     ∂p_reversediff = ReverseDiff.gradient(solve_nlprob, p)
-
-    @test ∂p_zygote ≈ ∂p_forwarddiff ≈ ∂p_tracker ≈ ∂p_reversediff
+    @test ∂p_zygote ≈ ∂p_tracker ≈ ∂p_reversediff
+    #@test ∂p_zygote ≈ ∂p_forwarddiff ≈ ∂p_tracker ≈ ∂p_reversediff
 end
diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
index efc03403a..5050b702d 100644
--- a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
@@ -51,6 +51,7 @@ end
     end
 
     prob = NonlinearProblem{false}(f, @SVector[1.0f0, 1.0f0])
+    prob = convert(SimpleNonlinearSolve.ImmutableNonlinearProblem, prob)
 
     @testset "$(nameof(typeof(alg)))" for alg in (
         SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(),

From 96b40e1d6970c0aeda9671fe92ef76b4b7c32a8d Mon Sep 17 00:00:00 2001
From: Matt Bossart <bossart.matt@gmail.com>
Date: Wed, 17 Jul 2024 19:05:43 -0400
Subject: [PATCH 349/375] convert to immutable in ad dispatch

---
 lib/SimpleNonlinearSolve/src/ad.jl | 35 ++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 353cc0b9d..ce291d82e 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -1,15 +1,26 @@
-for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem)
-    @eval function SciMLBase.solve(
-            prob::$(pType){<:Union{Number, <:AbstractArray}, iip,
-                <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
-            alg::AbstractSimpleNonlinearSolveAlgorithm,
-            args...;
-            kwargs...) where {T, V, P, iip}
-        sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
-        dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
-        return SciMLBase.build_solution(
-            prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
-    end
+function SciMLBase.solve(
+        prob::NonlinearLeastSquaresProblem{<:Union{Number, <:AbstractArray}, iip,
+            <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...;
+        kwargs...) where {T, V, P, iip}
+    sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
+    dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
+    return SciMLBase.build_solution(
+        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
+end
+
+function SciMLBase.solve(
+        prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip,
+            <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...;
+        kwargs...) where {T, V, P, iip}
+    prob = convert(ImmutableNonlinearProblem, prob)
+    sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
+    dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
+    return SciMLBase.build_solution(
+        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
 end
 
 for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)

From 62c928e779159491f768bc78bb25f1122b55c0ea Mon Sep 17 00:00:00 2001
From: Matt Bossart <67015312+m-bossart@users.noreply.github.com>
Date: Mon, 22 Jul 2024 08:50:43 -0400
Subject: [PATCH 350/375] Update test/core/adjoint_tests.jl

Co-authored-by: Christopher Rackauckas <accounts@chrisrackauckas.com>
---
 lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
index a4757f10d..351557930 100644
--- a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
@@ -13,7 +13,7 @@
     p = [3.0, 2.0]
 
     ∂p_zygote = only(Zygote.gradient(solve_nlprob, p))
-    #∂p_forwarddiff = ForwardDiff.gradient(solve_nlprob, p)
+    ∂p_forwarddiff = ForwardDiff.gradient(solve_nlprob, p)
     ∂p_tracker = Tracker.data(only(Tracker.gradient(solve_nlprob, p)))
     ∂p_reversediff = ReverseDiff.gradient(solve_nlprob, p)
     @test ∂p_zygote ≈ ∂p_tracker ≈ ∂p_reversediff

From d85408e7d1966980b9e406b79ac23ad6a65ed4a0 Mon Sep 17 00:00:00 2001
From: Matt Bossart <67015312+m-bossart@users.noreply.github.com>
Date: Mon, 22 Jul 2024 08:50:50 -0400
Subject: [PATCH 351/375] Update test/core/adjoint_tests.jl

Co-authored-by: Christopher Rackauckas <accounts@chrisrackauckas.com>
---
 lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
index 351557930..9b1ae05ff 100644
--- a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
@@ -17,5 +17,5 @@
     ∂p_tracker = Tracker.data(only(Tracker.gradient(solve_nlprob, p)))
     ∂p_reversediff = ReverseDiff.gradient(solve_nlprob, p)
     @test ∂p_zygote ≈ ∂p_tracker ≈ ∂p_reversediff
-    #@test ∂p_zygote ≈ ∂p_forwarddiff ≈ ∂p_tracker ≈ ∂p_reversediff
+    @test ∂p_zygote ≈ ∂p_forwarddiff ≈ ∂p_tracker ≈ ∂p_reversediff
 end

From b50726f477a81f32eeb546540e72b9572711816d Mon Sep 17 00:00:00 2001
From: Matt Bossart <67015312+m-bossart@users.noreply.github.com>
Date: Mon, 22 Jul 2024 08:51:39 -0400
Subject: [PATCH 352/375] Update src/ad.jl

Co-authored-by: Christopher Rackauckas <accounts@chrisrackauckas.com>
---
 lib/SimpleNonlinearSolve/src/ad.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index ce291d82e..3d9ce8d0e 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -42,7 +42,7 @@ for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
 end
 
 function __nlsolve_ad(
-        prob::Union{IntervalNonlinearProblem, ImmutableNonlinearProblem}, alg, args...; kwargs...)
+        prob::Union{IntervalNonlinearProblem, NonlinearProblem, ImmutableNonlinearProblem}, alg, args...; kwargs...)
     p = value(prob.p)
     if prob isa IntervalNonlinearProblem
         tspan = value.(prob.tspan)

From 221759ee4df730ced1dade9bf4d808162f4c99ba Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 22 Jul 2024 10:10:33 -0400
Subject: [PATCH 353/375] format

---
 .../src/SimpleNonlinearSolve.jl               | 17 ++--
 lib/SimpleNonlinearSolve/src/ad.jl            |  3 +-
 .../src/immutable_nonlinear_problem.jl        | 77 +++++++++----------
 .../src/nlsolve/lbroyden.jl                   |  6 +-
 .../src/nlsolve/raphson.jl                    |  3 +-
 .../src/nlsolve/trustRegion.jl                |  4 +-
 lib/SimpleNonlinearSolve/src/utils.jl         |  6 +-
 7 files changed, 59 insertions(+), 57 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index 0baa27655..ba0b243af 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -20,10 +20,10 @@ using LinearAlgebra: LinearAlgebra, I, convert, copyto!, diagind, dot, issuccess
 using MaybeInplace: @bb, setindex_trait, CanSetindex, CannotSetindex
 using Reexport: @reexport
 using SciMLBase: @add_kwonly, SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem,
-                 AbstractNonlinearFunction, StandardNonlinearProblem,
-                 NonlinearFunction, NonlinearLeastSquaresProblem, NonlinearProblem,
-                 ReturnCode, init, remake, solve, AbstractNonlinearAlgorithm,
-                 build_solution, isinplace, _unwrap_val, warn_paramtype
+                 AbstractNonlinearFunction, StandardNonlinearProblem, NonlinearFunction,
+                 NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode, init, remake,
+                 solve, AbstractNonlinearAlgorithm, build_solution, isinplace, _unwrap_val,
+                 warn_paramtype
 using Setfield: @set!
 using StaticArraysCore: StaticArray, SVector, SMatrix, SArray, MArray, Size
 
@@ -81,8 +81,9 @@ function SciMLBase.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSol
         p === nothing, alg, args...; prob.kwargs..., kwargs...)
 end
 
-function SciMLBase.solve(prob::ImmutableNonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
-    args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...)
+function SciMLBase.solve(
+        prob::ImmutableNonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...)
     if sensealg === nothing && haskey(prob.kwargs, :sensealg)
         sensealg = prob.kwargs[:sensealg]
     end
@@ -92,8 +93,8 @@ function SciMLBase.solve(prob::ImmutableNonlinearProblem, alg::AbstractSimpleNon
         p === nothing, alg, args...; prob.kwargs..., kwargs...)
 end
 
-function __internal_solve_up(_prob::ImmutableNonlinearProblem, sensealg, u0, u0_changed,
-        p, p_changed, alg, args...; kwargs...)
+function __internal_solve_up(_prob::ImmutableNonlinearProblem, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
     prob = u0_changed || p_changed ? remake(_prob; u0, p) : _prob
     return SciMLBase.__solve(prob, alg, args...; kwargs...)
 end
diff --git a/lib/SimpleNonlinearSolve/src/ad.jl b/lib/SimpleNonlinearSolve/src/ad.jl
index 3d9ce8d0e..a83abdea7 100644
--- a/lib/SimpleNonlinearSolve/src/ad.jl
+++ b/lib/SimpleNonlinearSolve/src/ad.jl
@@ -42,7 +42,8 @@ for algType in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
 end
 
 function __nlsolve_ad(
-        prob::Union{IntervalNonlinearProblem, NonlinearProblem, ImmutableNonlinearProblem}, alg, args...; kwargs...)
+        prob::Union{IntervalNonlinearProblem, NonlinearProblem, ImmutableNonlinearProblem},
+        alg, args...; kwargs...)
     p = value(prob.p)
     if prob isa IntervalNonlinearProblem
         tspan = value.(prob.tspan)
diff --git a/lib/SimpleNonlinearSolve/src/immutable_nonlinear_problem.jl b/lib/SimpleNonlinearSolve/src/immutable_nonlinear_problem.jl
index ca1dfc079..99772c5b0 100644
--- a/lib/SimpleNonlinearSolve/src/immutable_nonlinear_problem.jl
+++ b/lib/SimpleNonlinearSolve/src/immutable_nonlinear_problem.jl
@@ -1,65 +1,60 @@
 struct ImmutableNonlinearProblem{uType, isinplace, P, F, K, PT} <:
-    AbstractNonlinearProblem{uType, isinplace}
- f::F
- u0::uType
- p::P
- problem_type::PT
- kwargs::K
- @add_kwonly function ImmutableNonlinearProblem{iip}(f::AbstractNonlinearFunction{iip}, u0,
-         p = NullParameters(),
-         problem_type = StandardNonlinearProblem();
-         kwargs...) where {iip}
-     if haskey(kwargs, :p)
-         error("`p` specified as a keyword argument `p = $(kwargs[:p])` to `NonlinearProblem`. This is not supported.")
-     end
-     warn_paramtype(p)
-     new{typeof(u0), iip, typeof(p), typeof(f),
-         typeof(kwargs), typeof(problem_type)}(f,
-         u0,
-         p,
-         problem_type,
-         kwargs)
- end
+       AbstractNonlinearProblem{uType, isinplace}
+    f::F
+    u0::uType
+    p::P
+    problem_type::PT
+    kwargs::K
+    @add_kwonly function ImmutableNonlinearProblem{iip}(
+            f::AbstractNonlinearFunction{iip}, u0, p = NullParameters(),
+            problem_type = StandardNonlinearProblem(); kwargs...) where {iip}
+        if haskey(kwargs, :p)
+            error("`p` specified as a keyword argument `p = $(kwargs[:p])` to `NonlinearProblem`. This is not supported.")
+        end
+        warn_paramtype(p)
+        new{typeof(u0), iip, typeof(p), typeof(f), typeof(kwargs), typeof(problem_type)}(
+            f, u0, p, problem_type, kwargs)
+    end
 
- """
- Define a steady state problem using the given function.
- `isinplace` optionally sets whether the function is inplace or not.
- This is determined automatically, but not inferred.
- """
- function ImmutableNonlinearProblem{iip}(f, u0, p = NullParameters(); kwargs...) where {iip}
-     ImmutableNonlinearProblem{iip}(NonlinearFunction{iip}(f), u0, p; kwargs...)
- end
+    """
+    Define a steady state problem using the given function.
+    `isinplace` optionally sets whether the function is inplace or not.
+    This is determined automatically, but not inferred.
+    """
+    function ImmutableNonlinearProblem{iip}(
+            f, u0, p = NullParameters(); kwargs...) where {iip}
+        ImmutableNonlinearProblem{iip}(NonlinearFunction{iip}(f), u0, p; kwargs...)
+    end
 end
 
 """
 Define a nonlinear problem using an instance of
 [`AbstractNonlinearFunction`](@ref AbstractNonlinearFunction).
 """
-function ImmutableNonlinearProblem(f::AbstractNonlinearFunction, u0, p = NullParameters(); kwargs...)
- ImmutableNonlinearProblem{isinplace(f)}(f, u0, p; kwargs...)
+function ImmutableNonlinearProblem(
+        f::AbstractNonlinearFunction, u0, p = NullParameters(); kwargs...)
+    ImmutableNonlinearProblem{isinplace(f)}(f, u0, p; kwargs...)
 end
 
 function ImmutableNonlinearProblem(f, u0, p = NullParameters(); kwargs...)
- ImmutableNonlinearProblem(NonlinearFunction(f), u0, p; kwargs...)
+    ImmutableNonlinearProblem(NonlinearFunction(f), u0, p; kwargs...)
 end
 
 """
 Define a ImmutableNonlinearProblem problem from SteadyStateProblem
 """
 function ImmutableNonlinearProblem(prob::AbstractNonlinearProblem)
- ImmutableNonlinearProblem{isinplace(prob)}(prob.f, prob.u0, prob.p)
+    ImmutableNonlinearProblem{isinplace(prob)}(prob.f, prob.u0, prob.p)
 end
 
-
-function Base.convert(::Type{ImmutableNonlinearProblem}, prob::T) where {T <: NonlinearProblem}
- ImmutableNonlinearProblem{isinplace(prob)}(prob.f,
-     prob.u0,
-     prob.p,
-     prob.problem_type;
-     prob.kwargs...)
+function Base.convert(
+        ::Type{ImmutableNonlinearProblem}, prob::T) where {T <: NonlinearProblem}
+    ImmutableNonlinearProblem{isinplace(prob)}(
+        prob.f, prob.u0, prob.p, prob.problem_type; prob.kwargs...)
 end
 
-function DiffEqBase.get_concrete_problem(prob::ImmutableNonlinearProblem, isadapt; kwargs...)
+function DiffEqBase.get_concrete_problem(
+        prob::ImmutableNonlinearProblem, isadapt; kwargs...)
     u0 = DiffEqBase.get_concrete_u0(prob, isadapt, nothing, kwargs)
     u0 = DiffEqBase.promote_u0(u0, prob.p, nothing)
     p = DiffEqBase.get_concrete_p(prob, kwargs)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
index 683e9e592..891043aca 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/lbroyden.jl
@@ -44,7 +44,8 @@ function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleLimitedMe
     return __generic_solve(prob, alg, args...; termination_condition, kwargs...)
 end
 
-@views function __generic_solve(prob::ImmutableNonlinearProblem, alg::SimpleLimitedMemoryBroyden,
+@views function __generic_solve(
+        prob::ImmutableNonlinearProblem, alg::SimpleLimitedMemoryBroyden,
         args...; abstol = nothing, reltol = nothing, maxiters = 1000,
         alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
@@ -114,7 +115,8 @@ end
 # Non-allocating StaticArrays version of SimpleLimitedMemoryBroyden is actually quite
 # finicky, so we'll implement it separately from the generic version
 # Ignore termination_condition. Don't pass things into internal functions
-function __static_solve(prob::ImmutableNonlinearProblem{<:SArray}, alg::SimpleLimitedMemoryBroyden,
+function __static_solve(
+        prob::ImmutableNonlinearProblem{<:SArray}, alg::SimpleLimitedMemoryBroyden,
         args...; abstol = nothing, maxiters = 1000, kwargs...)
     x = prob.u0
     fx = _get_fx(prob, x)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
index d7e72dbdf..bdf45fffc 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/raphson.jl
@@ -23,7 +23,8 @@ end
 
 const SimpleGaussNewton = SimpleNewtonRaphson
 
-function SciMLBase.__solve(prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
+function SciMLBase.__solve(
+        prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
         alg::SimpleNewtonRaphson, args...; abstol = nothing, reltol = nothing,
         maxiters = 1000, termination_condition = nothing, alias_u0 = false, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
index 2e78baf02..cc937724c 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/trustRegion.jl
@@ -55,8 +55,8 @@ scalar and static array problems.
     nlsolve_update_rule = Val(false)
 end
 
-function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleTrustRegion, args...;
-        abstol = nothing, reltol = nothing, maxiters = 1000,
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleTrustRegion,
+        args...; abstol = nothing, reltol = nothing, maxiters = 1000,
         alias_u0 = false, termination_condition = nothing, kwargs...)
     x = __maybe_unaliased(prob.u0, alias_u0)
     T = eltype(real(x))
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 9380ff186..157d8f03f 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -145,7 +145,8 @@ end
 # different. NonlinearSolve is more for robust / cached solvers while SimpleNonlinearSolve
 # is meant for low overhead solvers, users can opt into the other termination modes but the
 # default is to use the least overhead version.
-function init_termination_cache(prob::ImmutableNonlinearProblem, abstol, reltol, du, u, ::Nothing)
+function init_termination_cache(
+        prob::ImmutableNonlinearProblem, abstol, reltol, du, u, ::Nothing)
     return init_termination_cache(
         prob, abstol, reltol, du, u, AbsNormTerminationMode(Base.Fix1(maximum, abs)))
 end
@@ -155,7 +156,8 @@ function init_termination_cache(
         prob, abstol, reltol, du, u, AbsNormTerminationMode(Base.Fix2(norm, 2)))
 end
 
-function init_termination_cache(prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
+function init_termination_cache(
+        prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
         abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode)
     T = promote_type(eltype(du), eltype(u))
     abstol = __get_tolerance(u, abstol, T)

From 5e5aed1d309eb439d5a93288c76f329bc94cc9cd Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Mon, 22 Jul 2024 10:15:59 -0400
Subject: [PATCH 354/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index c27d74903..8e7311f58 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.10.1"
+version = "1.11.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From 232e91e2630173daecaf4b01865e63af31f56803 Mon Sep 17 00:00:00 2001
From: Anant Thazhemadam <anant.thazhemadam@gmail.com>
Date: Mon, 29 Apr 2024 19:27:29 +0530
Subject: [PATCH 355/375] ci: update invalidations workflow to use centralised
 reusable workflow

---
 .../.github/workflows/Invalidations.yml       | 33 +++----------------
 1 file changed, 4 insertions(+), 29 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml b/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
index 66c86a362..34eb7a92a 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Invalidations.yml
@@ -1,4 +1,4 @@
-name: Invalidations
+name: "Invalidations"
 
 on:
   pull_request:
@@ -10,31 +10,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  evaluate:
-    # Only run on PRs to the default branch.
-    # In the PR trigger above branches can be specified only explicitly whereas this check should work for master, main, or any other default branch
-    if: github.base_ref == github.event.repository.default_branch
-    runs-on: ubuntu-latest
-    steps:
-    - uses: julia-actions/setup-julia@v2
-      with:
-        version: '1'
-    - uses: actions/checkout@v4
-    - uses: julia-actions/julia-buildpkg@v1
-    - uses: julia-actions/julia-invalidations@v1
-      id: invs_pr
-
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.repository.default_branch }}
-    - uses: julia-actions/julia-buildpkg@v1
-    - uses: julia-actions/julia-invalidations@v1
-      id: invs_default
-    
-    - name: Report invalidation counts
-      run: |
-        echo "Invalidations on default branch: ${{ steps.invs_default.outputs.total }} (${{ steps.invs_default.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY
-        echo "This branch: ${{ steps.invs_pr.outputs.total }} (${{ steps.invs_pr.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY
-    - name: Check if the PR does increase number of invalidations
-      if: steps.invs_pr.outputs.total > steps.invs_default.outputs.total
-      run: exit 1
+  evaluate-invalidations:
+    name: "Evaluate Invalidations"
+    uses: "SciML/.github/.github/workflows/invalidations.yml@v1"

From 35e07737955c0d5181af0cfe8fcd8afc01f9d7b6 Mon Sep 17 00:00:00 2001
From: Anant Thazhemadam <anant.thazhemadam@gmail.com>
Date: Mon, 29 Apr 2024 19:41:25 +0530
Subject: [PATCH 356/375] ci: update format check workflow to use centralised
 reusable workflow

---
 .../.github/workflows/FormatCheck.yml            | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
index 0ddeb4ed1..7e46c8db9 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
@@ -1,9 +1,13 @@
-name: Format suggestions
+name: "Format Check"
 
-on: [pull_request]
+on:
+  push:
+    branches:
+      - 'main'
+    tags: '*'
+  pull_request:
 
 jobs:
-  code-style:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: julia-actions/julia-format@v3
+  format-check:
+    name: "Format Check"
+    uses: "SciML/.github/.github/workflows/format-check.yml@v1"

From 231b11483aa5de737e2a111bbd70f0c9b4f293fb Mon Sep 17 00:00:00 2001
From: Anant Thazhemadam <anant.thazhemadam@gmail.com>
Date: Thu, 1 Aug 2024 00:12:14 +0200
Subject: [PATCH 357/375] ci: update tests workflow to use centralized reusable
 workflow

---
 .../.github/workflows/CI.yml                  | 55 -------------------
 .../.github/workflows/Tests.yml               | 42 ++++++++++++++
 2 files changed, 42 insertions(+), 55 deletions(-)
 delete mode 100644 lib/SimpleNonlinearSolve/.github/workflows/CI.yml
 create mode 100644 lib/SimpleNonlinearSolve/.github/workflows/Tests.yml

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml b/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
deleted file mode 100644
index 295de1024..000000000
--- a/lib/SimpleNonlinearSolve/.github/workflows/CI.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-name: CI
-on:
-  pull_request:
-    branches:
-      - main
-  push:
-    branches:
-      - main
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        group:
-          - Core
-        version:
-          - '1'
-        os:
-          - ubuntu-latest
-          - macos-latest
-          - windows-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@v2
-        with:
-          version: ${{ matrix.version }}
-      - uses: actions/cache@v4
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-
-            ${{ runner.os }}-
-      - uses: julia-actions/julia-buildpkg@v1
-      - uses: julia-actions/julia-runtest@v1
-        with:
-          annotate: true
-        env:
-          GROUP: ${{ matrix.group }}
-          JULIA_NUM_THREADS: 11
-          RETESTITEMS_NWORKERS: 4
-          RETESTITEMS_NWORKER_THREADS: 2
-      - uses: julia-actions/julia-processcoverage@v1
-        with:
-          directories: src,ext
-      - uses: codecov/codecov-action@v4
-        with:
-          file: lcov.info
-          token: ${{ secrets.CODECOV_TOKEN }}
-          verbose: true
-          fail_ci_if_error: true
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml b/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml
new file mode 100644
index 000000000..8fa495e19
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml
@@ -0,0 +1,42 @@
+name: "Tests"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - 'release-'
+    paths-ignore:
+      - 'docs/**'
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - 'docs/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref_name != github.event.repository.default_branch || github.ref != 'refs/tags/v*' }}
+
+env:
+  JULIA_NUM_THREADS: 11
+  RETESTITEMS_NWORKERS: 4
+  RETESTITEMS_NWORKER_THREADS: 2
+
+jobs:
+  tests:
+    name: "Tests"
+    strategy:
+      fail-fast: false
+      matrix:
+        group:
+          - "Core"
+        os:
+          - "ubuntu-latest"
+          - "macos-latest"
+          - "windows-latest"
+
+    uses: "SciML/.github/.github/workflows/tests.yml@v1"
+    with:
+      group: "${{ matrix.group }}"
+      os: "${{ matrix.os }}"
+    secrets: "inherit"

From eb43c5a4cda6801107cde57d6f00c82b3c744def Mon Sep 17 00:00:00 2001
From: Anant Thazhemadam <anant.thazhemadam@gmail.com>
Date: Thu, 1 Aug 2024 00:12:26 +0200
Subject: [PATCH 358/375] ci(format-check): automatically comment formatting
 suggestions on PRs

---
 lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
index 7e46c8db9..fe9c1280f 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/FormatCheck.yml
@@ -10,4 +10,4 @@ on:
 jobs:
   format-check:
     name: "Format Check"
-    uses: "SciML/.github/.github/workflows/format-check.yml@v1"
+    uses: "SciML/.github/.github/workflows/format-suggestions-on-pr.yml@v1"

From 2e63a1346e8209e91528ca1a6ab7dee3903965ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez?= <daniel.glez.arribas@gmail.com>
Date: Fri, 9 Aug 2024 16:37:08 +0200
Subject: [PATCH 359/375] Docs, scaling factor

---
 lib/SimpleNonlinearSolve/src/bracketing/itp.jl | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index 4a542a958..e75642c5b 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -15,16 +15,22 @@ I. F. D. Oliveira and R. H. C. Takahashi.
 The following keyword parameters are accepted.
 
   - `n₀::Int = 10`, the 'slack'. Must not be negative. When n₀ = 0 the worst-case is
-    identical to that of bisection, but increacing n₀ provides greater oppotunity for
+    identical to that of bisection, but increasing n₀ provides greater opportunity for
     superlinearity.
-  - `scaled_κ₁::Float64 = 0.2`. Must not be negative. The recomended value is `0.2`.
+  - `scaled_κ₁::Float64 = 0.2`. Must not be negative. The recommended value is `0.2`.
     Lower values produce tighter asymptotic behaviour, while higher values improve the
     steady-state behaviour when truncation is not helpful.
   - `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62). Higher values allow for a greater
     convergence rate, but also make the method more succeptable to worst-case performance.
-    In practice, κ=1, 2 seems to work well due to the computational simplicity, as κ₂ is
+    In practice, κ₂=1, 2 seems to work well due to the computational simplicity, as κ₂ is
     used as an exponent in the method.
 
+### Computation of κ₁
+
+In the current implementation, we compute κ₁ = scaled_κ₁·|Δx₀|^(1 - κ₂); this allows κ₁ to
+adapt to the dimension of the problem in order to keep the proposed initial step
+proportional to Δx₀.
+
 ### Worst Case Performance
 
 n½ + `n₀` iterations, where n½ is the number of iterations using bisection
@@ -72,8 +78,8 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
     end
     ϵ = abstol
     #defining variables/cache
-    k1 = alg.scaled_k1 / abs(right - left)
     k2 = alg.k2
+    k1 = alg.scaled_k1 * abs(right - left)^(1 - k2)
     n0 = alg.n0
     n_h = ceil(log2(abs(right - left) / (2 * ϵ)))
     mid = (left + right) / 2

From ec1e309598547a20fc6006551cd416dbad78d01e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez?= <daniel.glez.arribas@gmail.com>
Date: Fri, 9 Aug 2024 17:24:46 +0200
Subject: [PATCH 360/375] Docstring update

---
 lib/SimpleNonlinearSolve/src/bracketing/itp.jl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index e75642c5b..4cc9e9b0c 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -28,8 +28,7 @@ The following keyword parameters are accepted.
 ### Computation of κ₁
 
 In the current implementation, we compute κ₁ = scaled_κ₁·|Δx₀|^(1 - κ₂); this allows κ₁ to
-adapt to the dimension of the problem in order to keep the proposed initial step
-proportional to Δx₀.
+adapt to the length of the interval and keep the proposed steps proportional to Δx.
 
 ### Worst Case Performance
 

From 145d5f221c5b4c43c24436b79390fe1b1c881620 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez?= <daniel.glez.arribas@gmail.com>
Date: Fri, 9 Aug 2024 18:09:38 +0200
Subject: [PATCH 361/375] Fix ITP for generic output numbers

---
 lib/SimpleNonlinearSolve/src/bracketing/itp.jl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index 4cc9e9b0c..4c92645ba 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -124,10 +124,11 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
         xp <= tmin && (xp = nextfloat(tmin))
         yp = f(xp)
         yps = yp * sign(fr)
-        if yps > 0
+        T0 = zero(yps)
+        if yps > T0
             right = xp
             fr = yp
-        elseif yps < 0
+        elseif yps < T0
             left = xp
             fl = yp
         else

From 0614b6aff579dd30c9a799aa04fa2d99a5c22e6c Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Sat, 10 Aug 2024 11:46:43 -0400
Subject: [PATCH 362/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 8e7311f58..c90abe15c 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.11.0"
+version = "1.12.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From e799b105ee7439abc3323ba625284e46fae53219 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Mon, 16 Sep 2024 17:14:17 -0400
Subject: [PATCH 363/375] fix: missing Tracker.data

---
 lib/SimpleNonlinearSolve/Project.toml                          | 2 +-
 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index c90abe15c..030b0385e 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.12.0"
+version = "1.12.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
index 679274754..fd56ee284 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
@@ -37,7 +37,7 @@ for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem)
                 prob, sensealg, u0, p, TrackerOriginator(), alg, args...; kwargs...)
 
             function ∇__internal_solve_up(Δ)
-                ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Δ)
+                ∂prob, ∂sensealg, ∂u0, ∂p, ∂originator, ∂args... = ∇internal(Tracker.data(Δ))
                 return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
             end
 

From 7b994727afd2df965fa2d112f1f5d9fe87779a53 Mon Sep 17 00:00:00 2001
From: tom <tom.rottier@manchester.ac.uk>
Date: Mon, 23 Sep 2024 18:09:24 -0400
Subject: [PATCH 364/375] Return InitialFailure from bracketing methods if not
 enclosing interval

---
 lib/SimpleNonlinearSolve/src/bracketing/bisection.jl | 6 ++++++
 lib/SimpleNonlinearSolve/src/bracketing/brent.jl     | 6 ++++++
 lib/SimpleNonlinearSolve/src/bracketing/falsi.jl     | 6 ++++++
 lib/SimpleNonlinearSolve/src/bracketing/itp.jl       | 7 +++++++
 lib/SimpleNonlinearSolve/src/bracketing/ridder.jl    | 6 ++++++
 5 files changed, 31 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
index d55a0ce5e..adcf72a4b 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
@@ -39,6 +39,12 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection,
             prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
 
+    if sign(fl) == sign(fr)
+        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
     i = 1
     if !iszero(fr)
         while i < maxiters
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
index 649286e03..4ba311f5d 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
@@ -26,6 +26,12 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
             prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
 
+    if sign(fl) == sign(fr)
+        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
     if abs(fl) < abs(fr)
         c = right
         right = left
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
index ee78b73fc..00b29703b 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
@@ -25,6 +25,12 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
             prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
 
+    if sign(fl) == sign(fr)
+        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
     # Regula Falsi Steps
     i = 0
     if !iszero(fr)
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index 4c92645ba..9405cc2de 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -75,6 +75,13 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
         return build_solution(
             prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
+
+    if sign(fl) == sign(fr)
+        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
     ϵ = abstol
     #defining variables/cache
     k2 = alg.k2
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
index a974824c2..772f568d1 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
@@ -25,6 +25,12 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
             prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
     end
 
+    if sign(fl) == sign(fr)
+        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        return build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
     xo = oftype(left, Inf)
     i = 1
     if !iszero(fr)

From 45d63d20891170ba7f671b44c79b21e0c22854cb Mon Sep 17 00:00:00 2001
From: Christopher Rackauckas <accounts@chrisrackauckas.com>
Date: Tue, 24 Sep 2024 06:22:05 -0400
Subject: [PATCH 365/375] Update Project.toml

---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 030b0385e..8d7b044a9 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.12.1"
+version = "1.12.2"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"

From 5742f89cb3049e1fb4feedc0b932ef6eb8b71f03 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 25 Sep 2024 14:57:03 -0400
Subject: [PATCH 366/375] feat: update to new DI.jl

---
 lib/SimpleNonlinearSolve/Project.toml | 6 +++---
 lib/SimpleNonlinearSolve/src/utils.jl | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 8d7b044a9..5927f6fe2 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -1,7 +1,7 @@
 name = "SimpleNonlinearSolve"
 uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 authors = ["SciML"]
-version = "1.12.2"
+version = "1.12.3"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
@@ -34,7 +34,7 @@ SimpleNonlinearSolveTrackerExt = "Tracker"
 SimpleNonlinearSolveZygoteExt = "Zygote"
 
 [compat]
-ADTypes = "1.2"
+ADTypes = "1.9"
 AllocCheck = "0.1.1"
 Aqua = "0.8"
 ArrayInterface = "7.9"
@@ -43,7 +43,7 @@ ChainRulesCore = "1.23"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.149"
 DiffResults = "1.1"
-DifferentiationInterface = "0.5"
+DifferentiationInterface = "0.6"
 ExplicitImports = "1.5.0"
 FastClosures = "0.3.2"
 FiniteDiff = "2.23.1"
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
index 157d8f03f..5e967fa7f 100644
--- a/lib/SimpleNonlinearSolve/src/utils.jl
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -28,7 +28,7 @@ end
 
 function value_and_jacobian(
         ad, prob::AbstractNonlinearProblem, f::F, y, x, cache; J = nothing) where {F}
-    x isa Number && return DI.value_and_derivative(f, ad, x, cache)
+    x isa Number && return DI.value_and_derivative(f, cache, ad, x)
 
     if isinplace(prob)
         if cache isa HasAnalyticJacobian
@@ -36,11 +36,11 @@ function value_and_jacobian(
             f(y, x)
             return y, J
         end
-        return DI.value_and_jacobian!(f, y, J, ad, x, cache)
+        return DI.value_and_jacobian!(f, y, J, cache, ad, x)
     else
         cache isa HasAnalyticJacobian && return f(x), prob.f.jac(x, prob.p)
-        J === nothing && return DI.value_and_jacobian(f, ad, x, cache)
-        y, J = DI.value_and_jacobian!(f, J, ad, x, cache)
+        J === nothing && return DI.value_and_jacobian(f, cache, ad, x)
+        y, J = DI.value_and_jacobian!(f, J, cache, ad, x)
         return y, J
     end
 end

From bd928936749e8ce140f1be49a2ae9bef1a02ebe2 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Wed, 25 Sep 2024 17:04:24 -0400
Subject: [PATCH 367/375] fix: install some test deps if needed

---
 .../.github/workflows/Tests.yml               |   3 +-
 lib/SimpleNonlinearSolve/Project.toml         |  12 +-
 .../test/core/23_test_problems_tests.jl       |   2 +-
 .../test/core/adjoint_tests.jl                |   2 +-
 .../test/core/rootfind_tests.jl               |   3 +-
 .../test/gpu/cuda_tests.jl                    | 108 +++++++++---------
 lib/SimpleNonlinearSolve/test/runtests.jl     |  31 +++--
 7 files changed, 90 insertions(+), 71 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml b/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml
index 8fa495e19..e63c1039e 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml
@@ -29,7 +29,8 @@ jobs:
       fail-fast: false
       matrix:
         group:
-          - "Core"
+          - "core"
+          - "adjoint"
         os:
           - "ubuntu-latest"
           - "macos-latest"
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 5927f6fe2..444bfed89 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -38,7 +38,6 @@ ADTypes = "1.9"
 AllocCheck = "0.1.1"
 Aqua = "0.8"
 ArrayInterface = "7.9"
-CUDA = "5.2"
 ChainRulesCore = "1.23"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.149"
@@ -48,8 +47,9 @@ ExplicitImports = "1.5.0"
 FastClosures = "0.3.2"
 FiniteDiff = "2.23.1"
 ForwardDiff = "0.10.36"
+Hwloc = "3"
+InteractiveUtils = "<0.0.1, 1"
 LinearAlgebra = "1.10"
-LinearSolve = "2.30"
 MaybeInplace = "0.1.3"
 NonlinearProblemLibrary = "0.1.2"
 Pkg = "1.10"
@@ -60,7 +60,6 @@ ReTestItems = "1.23"
 Reexport = "1.2"
 ReverseDiff = "1.15.3"
 SciMLBase = "2.37.0"
-SciMLSensitivity = "7.58"
 Setfield = "1.1.1"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"
@@ -72,13 +71,13 @@ julia = "1.10"
 [extras]
 AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
-CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+Hwloc = "0e44f5e4-bd66-52a0-8798-143a42290a1d"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 NonlinearProblemLibrary = "b7050fa9-e91f-4b37-bcee-a89a063da141"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
@@ -86,11 +85,10 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
-SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["AllocCheck", "Aqua", "CUDA", "DiffEqBase", "ExplicitImports", "FiniteDiff", "ForwardDiff", "LinearAlgebra", "LinearSolve", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReTestItems", "Reexport", "ReverseDiff", "SciMLSensitivity", "StaticArrays", "Test", "Tracker", "Zygote"]
+test = ["AllocCheck", "Aqua", "DiffEqBase", "ExplicitImports", "FiniteDiff", "ForwardDiff", "Hwloc", "InteractiveUtils", "LinearAlgebra",  "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReTestItems", "Reexport", "ReverseDiff",  "StaticArrays", "Test", "Tracker", "Zygote"]
diff --git a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
index e45560912..6d1980810 100644
--- a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
@@ -86,7 +86,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "23 Test Problems: SimpleBroyden" retries=5 setup=[RobustnessTesting] tags=[:core] begin
+@testitem "23 Test Problems: SimpleBroyden" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (SimpleBroyden(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
diff --git a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
index 9b1ae05ff..c56850eb5 100644
--- a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Simple Adjoint Test" tags=[:core] begin
+@testitem "Simple Adjoint Test" tags=[:adjoint] begin
     using ForwardDiff, ReverseDiff, SciMLSensitivity, Tracker, Zygote
 
     ff(u, p) = u .^ 2 .- p
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index eecdeca18..272fa3c4d 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -1,7 +1,6 @@
 @testsetup module RootfindingTesting
 using Reexport
-@reexport using AllocCheck, LinearSolve, StaticArrays, Random, LinearAlgebra, ForwardDiff,
-                DiffEqBase
+@reexport using AllocCheck, StaticArrays, Random, LinearAlgebra, ForwardDiff, DiffEqBase
 import PolyesterForwardDiff
 
 quadratic_f(u, p) = u .* u .- p
diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
index 5050b702d..fc99bddaf 100644
--- a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
@@ -1,73 +1,77 @@
-@testitem "Solving on GPUs" tags=[:cuda] skip=:(using CUDA; !CUDA.functional()) begin
+@testitem "Solving on GPUs" tags=[:cuda] begin
     using StaticArrays, CUDA
 
-    CUDA.allowscalar(false)
+    if CUDA.functional()
+        CUDA.allowscalar(false)
 
-    f(u, p) = u .* u .- 2
-    f!(du, u, p) = du .= u .* u .- 2
+        f(u, p) = u .* u .- 2
+        f!(du, u, p) = du .= u .* u .- 2
 
-    @testset "$(nameof(typeof(alg)))" for alg in (
-        SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(),
-        SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
-        SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
-        SimpleHalley(), SimpleBroyden(; linesearch = Val(true)),
-        SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
-        # Static Arrays
-        u0 = @SVector[1.0f0, 1.0f0]
-        probN = NonlinearProblem{false}(f, u0)
-        sol = solve(probN, alg; abstol = 1.0f-6)
-        @test SciMLBase.successful_retcode(sol)
-        @test maximum(abs, sol.resid) ≤ 1.0f-6
-
-        # Regular Arrays
-        u0 = [1.0, 1.0]
-        probN = NonlinearProblem{false}(f, u0)
-        sol = solve(probN, alg; abstol = 1.0f-6)
-        @test SciMLBase.successful_retcode(sol)
-        @test maximum(abs, sol.resid) ≤ 1.0f-6
+        @testset "$(nameof(typeof(alg)))" for alg in (
+            SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(),
+            SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+            SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
+            SimpleHalley(), SimpleBroyden(; linesearch = Val(true)),
+            SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
+            # Static Arrays
+            u0 = @SVector[1.0f0, 1.0f0]
+            probN = NonlinearProblem{false}(f, u0)
+            sol = solve(probN, alg; abstol = 1.0f-6)
+            @test SciMLBase.successful_retcode(sol)
+            @test maximum(abs, sol.resid) ≤ 1.0f-6
 
-        # Regular Arrays Inplace
-        if !(alg isa SimpleHalley)
+            # Regular Arrays
             u0 = [1.0, 1.0]
-            probN = NonlinearProblem{true}(f!, u0)
+            probN = NonlinearProblem{false}(f, u0)
             sol = solve(probN, alg; abstol = 1.0f-6)
             @test SciMLBase.successful_retcode(sol)
             @test maximum(abs, sol.resid) ≤ 1.0f-6
+
+            # Regular Arrays Inplace
+            if !(alg isa SimpleHalley)
+                u0 = [1.0, 1.0]
+                probN = NonlinearProblem{true}(f!, u0)
+                sol = solve(probN, alg; abstol = 1.0f-6)
+                @test SciMLBase.successful_retcode(sol)
+                @test maximum(abs, sol.resid) ≤ 1.0f-6
+            end
         end
     end
 end
 
-@testitem "CUDA Kernel Launch Test" tags=[:cuda] skip=:(using CUDA; !CUDA.functional()) timeout=3600 begin
+@testitem "CUDA Kernel Launch Test" tags=[:cuda] begin
     using StaticArrays, CUDA
 
-    CUDA.allowscalar(false)
+    if CUDA.functional()
+        CUDA.allowscalar(false)
 
-    f(u, p) = u .* u .- 2
-    f!(du, u, p) = du .= u .* u .- 2
+        f(u, p) = u .* u .- 2
+        f!(du, u, p) = du .= u .* u .- 2
 
-    function kernel_function(prob, alg)
-        solve(prob, alg)
-        return nothing
-    end
+        function kernel_function(prob, alg)
+            solve(prob, alg)
+            return nothing
+        end
 
-    prob = NonlinearProblem{false}(f, @SVector[1.0f0, 1.0f0])
-    prob = convert(SimpleNonlinearSolve.ImmutableNonlinearProblem, prob)
+        prob = NonlinearProblem{false}(f, @SVector[1.0f0, 1.0f0])
+        prob = convert(SimpleNonlinearSolve.ImmutableNonlinearProblem, prob)
 
-    @testset "$(nameof(typeof(alg)))" for alg in (
-        SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(),
-        SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
-        SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
-        SimpleHalley(), SimpleBroyden(; linesearch = Val(true)),
-        SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
-        @test begin
-            try
-                @cuda kernel_function(prob, alg)
-                @info "Successfully launched kernel for $(alg)."
-                true
-            catch err
-                @error "Kernel Launch failed for $(alg)."
-                false
-            end
-        end broken=(alg isa SimpleHalley)
+        @testset "$(nameof(typeof(alg)))" for alg in (
+            SimpleNewtonRaphson(), SimpleDFSane(), SimpleTrustRegion(),
+            SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+            SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(),
+            SimpleHalley(), SimpleBroyden(; linesearch = Val(true)),
+            SimpleLimitedMemoryBroyden(; linesearch = Val(true)))
+            @test begin
+                try
+                    @cuda kernel_function(prob, alg)
+                    @info "Successfully launched kernel for $(alg)."
+                    true
+                catch err
+                    @error "Kernel Launch failed for $(alg)."
+                    false
+                end
+            end broken=(alg isa SimpleHalley)
+        end
     end
 end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
index 2f81c8401..c90b994a4 100644
--- a/lib/SimpleNonlinearSolve/test/runtests.jl
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -1,10 +1,27 @@
-using ReTestItems, CUDA
+using ReTestItems, SimpleNonlinearSolve, Hwloc, InteractiveUtils
+using Pkg
 
-const GROUP = get(ENV, "GROUP", CUDA.functional() ? "All" : "Core")
+@info sprint(InteractiveUtils.versioninfo)
 
-if GROUP == "All"
-    ReTestItems.runtests(@__DIR__)
-else
-    tags = [Symbol(lowercase(GROUP))]
-    ReTestItems.runtests(@__DIR__; tags)
+const GROUP = lowercase(get(ENV, "GROUP", "All"))
+
+if GROUP == "adjoint" || GROUP == "all"
+    Pkg.add(["SciMLSensitivity"])
+end
+
+if GROUP == "cuda" || GROUP == "all"
+    Pkg.add(["CUDA"])
 end
+
+const RETESTITEMS_NWORKERS = parse(
+    Int, get(ENV, "RETESTITEMS_NWORKERS", string(min(Hwloc.num_physical_cores(), 4))))
+const RETESTITEMS_NWORKER_THREADS = parse(Int,
+    get(ENV, "RETESTITEMS_NWORKER_THREADS",
+        string(max(Hwloc.num_virtual_cores() ÷ RETESTITEMS_NWORKERS, 1))))
+
+@info "Running tests for group: $(GROUP) with $(RETESTITEMS_NWORKERS) workers"
+
+ReTestItems.runtests(
+    SimpleNonlinearSolve; tags = (GROUP == "all" ? nothing : [Symbol(GROUP)]),
+    nworkers = RETESTITEMS_NWORKERS, nworker_threads = RETESTITEMS_NWORKER_THREADS,
+    testitem_timeout = 3600, retries = 4)

From 96716b60a1a0e97b0cefbb1afca61f96294b5648 Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 26 Sep 2024 03:19:34 -0400
Subject: [PATCH 368/375] chore: update min DI version

Co-authored-by: Guillaume Dalle <22795598+gdalle@users.noreply.github.com>
---
 lib/SimpleNonlinearSolve/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 444bfed89..33e36201d 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -42,7 +42,7 @@ ChainRulesCore = "1.23"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.149"
 DiffResults = "1.1"
-DifferentiationInterface = "0.6"
+DifferentiationInterface = "0.6.1"
 ExplicitImports = "1.5.0"
 FastClosures = "0.3.2"
 FiniteDiff = "2.23.1"

From c77cee8c21a78e8d4f4a6f8da11a63779ea83a19 Mon Sep 17 00:00:00 2001
From: Adam Wheeler <awheeler@flatironinstitute.org>
Date: Fri, 27 Sep 2024 08:44:53 -0400
Subject: [PATCH 369/375] tweak warning message

---
 lib/SimpleNonlinearSolve/src/bracketing/bisection.jl | 2 +-
 lib/SimpleNonlinearSolve/src/bracketing/brent.jl     | 2 +-
 lib/SimpleNonlinearSolve/src/bracketing/falsi.jl     | 2 +-
 lib/SimpleNonlinearSolve/src/bracketing/itp.jl       | 2 +-
 lib/SimpleNonlinearSolve/src/bracketing/ridder.jl    | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
index adcf72a4b..75069ae17 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/bisection.jl
@@ -40,7 +40,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Bisection,
     end
 
     if sign(fl) == sign(fr)
-        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        @warn "The interval is not an enclosing interval (does not contain a root). Returning boundary value."
         return build_solution(
             prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
     end
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
index 4ba311f5d..ec208f939 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/brent.jl
@@ -27,7 +27,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
     end
 
     if sign(fl) == sign(fr)
-        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        @warn "The interval is not an enclosing interval (does not contain a root). Returning boundary value."
         return build_solution(
             prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
     end
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
index 00b29703b..ed1aceb9c 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/falsi.jl
@@ -26,7 +26,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
     end
 
     if sign(fl) == sign(fr)
-        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        @warn "The interval is not an enclosing interval (does not contain a root). Returning boundary value."
         return build_solution(
             prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
     end
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
index 9405cc2de..2cc98f076 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/itp.jl
@@ -77,7 +77,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
     end
 
     if sign(fl) == sign(fr)
-        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        @warn "The interval is not an enclosing interval (does not contain a root). Returning boundary value."
         return build_solution(
             prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
     end
diff --git a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
index 772f568d1..5c27a1077 100644
--- a/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
+++ b/lib/SimpleNonlinearSolve/src/bracketing/ridder.jl
@@ -26,7 +26,7 @@ function SciMLBase.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
     end
 
     if sign(fl) == sign(fr)
-        @warn "The interval is not an enclosing interval, opposite signs at the boundaries are required."
+        @warn "The interval is not an enclosing interval (does not contain a root). Returning boundary value."
         return build_solution(
             prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
     end

From b5f90257d2b6d1a344e0b464b2b1b916ec9608f9 Mon Sep 17 00:00:00 2001
From: Anant Thazhemadam <anant.thazhemadam@gmail.com>
Date: Thu, 17 Oct 2024 10:35:30 +0530
Subject: [PATCH 370/375] ci: test with `1`, `lts` and `pre` versions of julia

---
 lib/SimpleNonlinearSolve/.github/workflows/Tests.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml b/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml
index e63c1039e..c8c552d79 100644
--- a/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml
+++ b/lib/SimpleNonlinearSolve/.github/workflows/Tests.yml
@@ -28,6 +28,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        version:
+          - "1"
+          - "lts"
+          - "pre"
         group:
           - "core"
           - "adjoint"
@@ -38,6 +42,7 @@ jobs:
 
     uses: "SciML/.github/.github/workflows/tests.yml@v1"
     with:
+      julia-version: "${{ matrix.version }}"
       group: "${{ matrix.group }}"
       os: "${{ matrix.os }}"
     secrets: "inherit"

From a14a6998e90cf068ecc342ee8959959ac0fae5e4 Mon Sep 17 00:00:00 2001
From: Songchen Tan <i@tansongchen.com>
Date: Wed, 2 Oct 2024 15:41:34 -0400
Subject: [PATCH 371/375] Add Householder's method

---
 lib/SimpleNonlinearSolve/Project.toml         |  2 +
 .../ext/SimpleNonlinearSolveTaylorDiffExt.jl  | 60 +++++++++++++++++++
 .../src/SimpleNonlinearSolve.jl               |  2 +
 .../src/nlsolve/householder.jl                | 13 ++++
 4 files changed, 77 insertions(+)
 create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl
 create mode 100644 lib/SimpleNonlinearSolve/src/nlsolve/householder.jl

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 33e36201d..881ca6630 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -26,12 +26,14 @@ ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
+TaylorDiff = "b36ab563-344f-407b-a36a-4f200bebf99c"
 
 [extensions]
 SimpleNonlinearSolveChainRulesCoreExt = "ChainRulesCore"
 SimpleNonlinearSolveReverseDiffExt = "ReverseDiff"
 SimpleNonlinearSolveTrackerExt = "Tracker"
 SimpleNonlinearSolveZygoteExt = "Zygote"
+SimpleNonlinearSolveTaylorDiffExt = "TaylorDiff"
 
 [compat]
 ADTypes = "1.9"
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl
new file mode 100644
index 000000000..99b2ca788
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl
@@ -0,0 +1,60 @@
+module SimpleNonlinearSolveTaylorDiffExt
+using SimpleNonlinearSolve
+using SimpleNonlinearSolve: ImmutableNonlinearProblem, ReturnCode, build_solution, check_termination, init_termination_cache
+using SimpleNonlinearSolve: __maybe_unaliased, _get_fx, __fixed_parameter_function
+using MaybeInplace: @bb
+using SciMLBase: isinplace
+
+import TaylorDiff
+
+@inline function __get_higher_order_derivatives(::SimpleHouseholder{N}, prob, f, x) where N
+    vN = Val(N)
+    l = map(one, x)
+
+    if isinplace(prob)
+        fx = f(x)
+        invf = x -> inv.(f(x))
+        bundle = TaylorDiff.derivatives(invf, fx, x, l, vN)
+    else
+        t = TaylorDiff.make_seed(x, l, vN)
+        ft = f(t)
+        fx = map(TaylorDiff.primal, ft)
+        bundle = inv.(ft)
+    end
+    num = TaylorDiff.extract_derivative(bundle, N - 1)
+    den = TaylorDiff.extract_derivative(bundle, N)
+    return num, den, fx
+end
+
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleHouseholder{N},
+        args...; abstol = nothing, reltol = nothing, maxiters = 1000,
+        termination_condition = nothing, alias_u0 = false, kwargs...) where N
+    x = __maybe_unaliased(prob.u0, alias_u0)
+    length(x) == 1 ||
+        throw(ArgumentError("SimpleHouseholder only supports scalar problems"))
+    fx = _get_fx(prob, x)
+    @bb xo = copy(x)
+    f = __fixed_parameter_function(prob)
+
+    abstol, reltol, tc_cache = init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition)
+
+    for i in 1:maxiters
+        num, den, fx = __get_higher_order_derivatives(alg, prob, f, x)
+
+        if i == 1
+            iszero(fx) && build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+        else
+            # Termination Checks
+            tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg)
+            tc_sol !== nothing && return tc_sol
+        end
+
+        @bb copyto!(xo, x)
+        @bb x .+= (N - 1) .* num ./ den
+    end
+
+    return build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
index ba0b243af..e6c939b8f 100644
--- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -47,6 +47,7 @@ include("nlsolve/lbroyden.jl")
 include("nlsolve/klement.jl")
 include("nlsolve/trustRegion.jl")
 include("nlsolve/halley.jl")
+include("nlsolve/householder.jl")
 include("nlsolve/dfsane.jl")
 
 ## Interval Nonlinear Solvers
@@ -139,6 +140,7 @@ end
 export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff
 export SimpleBroyden, SimpleDFSane, SimpleGaussNewton, SimpleHalley, SimpleKlement,
        SimpleLimitedMemoryBroyden, SimpleNewtonRaphson, SimpleTrustRegion
+export SimpleHouseholder
 export Alefeld, Bisection, Brent, Falsi, ITP, Ridder
 
 end # module
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/householder.jl b/lib/SimpleNonlinearSolve/src/nlsolve/householder.jl
new file mode 100644
index 000000000..f012fff9d
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/householder.jl
@@ -0,0 +1,13 @@
+"""
+    SimpleHouseholder{order}()
+
+A low-overhead implementation of Householder's method. This method is non-allocating on scalar
+and static array problems.
+
+Internally, this uses TaylorDiff.jl for the automatic differentiation.
+
+### Type Parameters
+
+  - `order`: the convergence order of the Householder method. `order = 2` is the same as Newton's method, `order = 3` is the same as Halley's method, etc.
+"""
+struct SimpleHouseholder{order} <: AbstractNewtonAlgorithm end

From 229bdb58ee5ba8395de10865f1a02cd997e7f0e5 Mon Sep 17 00:00:00 2001
From: Songchen Tan <i@tansongchen.com>
Date: Tue, 8 Oct 2024 10:52:00 -0400
Subject: [PATCH 372/375] Add tests and doc

---
 .../ext/SimpleNonlinearSolveTaylorDiffExt.jl  | 18 ++++++-------
 .../src/nlsolve/householder.jl                |  9 ++++---
 .../test/core/rootfind_tests.jl               | 25 +++++++++++++++++++
 3 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl
index 99b2ca788..3257c6899 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl
@@ -7,20 +7,20 @@ using SciMLBase: isinplace
 
 import TaylorDiff
 
-@inline function __get_higher_order_derivatives(::SimpleHouseholder{N}, prob, f, x) where N
+@inline function __get_higher_order_derivatives(::SimpleHouseholder{N}, prob, f, x, fx) where N
     vN = Val(N)
     l = map(one, x)
+    t = TaylorDiff.make_seed(x, l, vN)
 
     if isinplace(prob)
-        fx = f(x)
-        invf = x -> inv.(f(x))
-        bundle = TaylorDiff.derivatives(invf, fx, x, l, vN)
+        bundle = similar(fx, TaylorDiff.TaylorScalar{eltype(fx), N})
+        f(bundle, t)
+        map!(TaylorDiff.primal, fx, bundle)
     else
-        t = TaylorDiff.make_seed(x, l, vN)
-        ft = f(t)
-        fx = map(TaylorDiff.primal, ft)
-        bundle = inv.(ft)
+        bundle = f(t)
+        fx = map(TaylorDiff.primal, bundle)
     end
+    bundle = inv.(bundle)
     num = TaylorDiff.extract_derivative(bundle, N - 1)
     den = TaylorDiff.extract_derivative(bundle, N)
     return num, den, fx
@@ -40,7 +40,7 @@ function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleHousehold
         prob, abstol, reltol, fx, x, termination_condition)
 
     for i in 1:maxiters
-        num, den, fx = __get_higher_order_derivatives(alg, prob, f, x)
+        num, den, fx = __get_higher_order_derivatives(alg, prob, f, x, fx)
 
         if i == 1
             iszero(fx) && build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
diff --git a/lib/SimpleNonlinearSolve/src/nlsolve/householder.jl b/lib/SimpleNonlinearSolve/src/nlsolve/householder.jl
index f012fff9d..ce06c5f3b 100644
--- a/lib/SimpleNonlinearSolve/src/nlsolve/householder.jl
+++ b/lib/SimpleNonlinearSolve/src/nlsolve/householder.jl
@@ -1,10 +1,13 @@
 """
     SimpleHouseholder{order}()
 
-A low-overhead implementation of Householder's method. This method is non-allocating on scalar
-and static array problems.
+A low-overhead implementation of Householder's method to arbitrary order.
+This method is non-allocating on scalar and static array problems.
 
-Internally, this uses TaylorDiff.jl for the automatic differentiation.
+!!! warning
+
+    Needs `TaylorDiff.jl` to be explicitly loaded before using this functionality.
+    Internally, this uses TaylorDiff.jl for automatic differentiation.
 
 ### Type Parameters
 
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index 272fa3c4d..76ba85d3c 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -91,6 +91,31 @@ end
     end
 end
 
+@testitem "SimpleHouseholder" setup=[RootfindingTesting] tags=[:core] begin
+    using TaylorDiff
+    @testset "AutoDiff: TaylorDiff.jl" for order in (2, 3, 4)
+        @testset "[OOP] u0: $(nameof(typeof(u0)))" for u0 in (
+            [1.0], @SVector[1.0], 1.0)
+            sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = SimpleHouseholder{order}())
+            @test SciMLBase.successful_retcode(sol)
+            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+        end
+
+        @testset "[IIP] u0: $(nameof(typeof(u0)))" for u0 in ([1.0],)
+            sol = benchmark_nlsolve_iip(quadratic_f!, u0; solver = SimpleHouseholder{order}())
+            @test SciMLBase.successful_retcode(sol)
+            @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
+        end
+    end
+
+    @testset "Termination condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+        u0 in (1.0, [1.0], @SVector[1.0])
+
+        probN = NonlinearProblem(quadratic_f, u0, 2.0)
+        @test all(solve(probN, SimpleHouseholder{2}(); termination_condition).u .≈ sqrt(2.0))
+    end
+end
+
 @testitem "Derivative Free Metods" setup=[RootfindingTesting] tags=[:core] begin
     @testset "$(nameof(typeof(alg)))" for alg in [
         SimpleBroyden(), SimpleKlement(), SimpleDFSane(),

From 32daa15155750bdd2a3ece3866ea088c20b4a93e Mon Sep 17 00:00:00 2001
From: Songchen Tan <i@tansongchen.com>
Date: Thu, 24 Oct 2024 15:27:48 -0400
Subject: [PATCH 373/375] Fix format and add compat

---
 lib/SimpleNonlinearSolve/Project.toml            |  4 +++-
 .../ext/SimpleNonlinearSolveTaylorDiffExt.jl     |  8 +++++---
 .../test/core/rootfind_tests.jl                  | 16 +++++++++-------
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index 881ca6630..e7f4f625c 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -65,6 +65,7 @@ SciMLBase = "2.37.0"
 Setfield = "1.1.1"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"
+TaylorDiff = "0.2.5"
 Test = "1.10"
 Tracker = "0.2.33"
 Zygote = "0.6.69"
@@ -88,9 +89,10 @@ ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+TaylorDiff = "b36ab563-344f-407b-a36a-4f200bebf99c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["AllocCheck", "Aqua", "DiffEqBase", "ExplicitImports", "FiniteDiff", "ForwardDiff", "Hwloc", "InteractiveUtils", "LinearAlgebra",  "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReTestItems", "Reexport", "ReverseDiff",  "StaticArrays", "Test", "Tracker", "Zygote"]
+test = ["AllocCheck", "Aqua", "DiffEqBase", "ExplicitImports", "FiniteDiff", "ForwardDiff", "Hwloc", "InteractiveUtils", "LinearAlgebra",  "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReTestItems", "Reexport", "ReverseDiff",  "StaticArrays", "TaylorDiff", "Test", "Tracker", "Zygote"]
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl
index 3257c6899..da096a74e 100644
--- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTaylorDiffExt.jl
@@ -1,13 +1,15 @@
 module SimpleNonlinearSolveTaylorDiffExt
 using SimpleNonlinearSolve
-using SimpleNonlinearSolve: ImmutableNonlinearProblem, ReturnCode, build_solution, check_termination, init_termination_cache
+using SimpleNonlinearSolve: ImmutableNonlinearProblem, ReturnCode, build_solution,
+                            check_termination, init_termination_cache
 using SimpleNonlinearSolve: __maybe_unaliased, _get_fx, __fixed_parameter_function
 using MaybeInplace: @bb
 using SciMLBase: isinplace
 
 import TaylorDiff
 
-@inline function __get_higher_order_derivatives(::SimpleHouseholder{N}, prob, f, x, fx) where N
+@inline function __get_higher_order_derivatives(
+        ::SimpleHouseholder{N}, prob, f, x, fx) where {N}
     vN = Val(N)
     l = map(one, x)
     t = TaylorDiff.make_seed(x, l, vN)
@@ -28,7 +30,7 @@ end
 
 function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleHouseholder{N},
         args...; abstol = nothing, reltol = nothing, maxiters = 1000,
-        termination_condition = nothing, alias_u0 = false, kwargs...) where N
+        termination_condition = nothing, alias_u0 = false, kwargs...) where {N}
     x = __maybe_unaliased(prob.u0, alias_u0)
     length(x) == 1 ||
         throw(ArgumentError("SimpleHouseholder only supports scalar problems"))
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
index 76ba85d3c..721d7fa90 100644
--- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -1,6 +1,7 @@
 @testsetup module RootfindingTesting
 using Reexport
-@reexport using AllocCheck, StaticArrays, Random, LinearAlgebra, ForwardDiff, DiffEqBase
+@reexport using AllocCheck, StaticArrays, Random, LinearAlgebra, ForwardDiff, DiffEqBase,
+                TaylorDiff
 import PolyesterForwardDiff
 
 quadratic_f(u, p) = u .* u .- p
@@ -92,17 +93,17 @@ end
 end
 
 @testitem "SimpleHouseholder" setup=[RootfindingTesting] tags=[:core] begin
-    using TaylorDiff
     @testset "AutoDiff: TaylorDiff.jl" for order in (2, 3, 4)
-        @testset "[OOP] u0: $(nameof(typeof(u0)))" for u0 in (
-            [1.0], @SVector[1.0], 1.0)
-            sol = benchmark_nlsolve_oop(quadratic_f, u0; solver = SimpleHouseholder{order}())
+        @testset "[OOP] u0: $(nameof(typeof(u0)))" for u0 in ([1.0], @SVector[1.0], 1.0)
+            sol = benchmark_nlsolve_oop(
+                quadratic_f, u0; solver = SimpleHouseholder{order}())
             @test SciMLBase.successful_retcode(sol)
             @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
         end
 
         @testset "[IIP] u0: $(nameof(typeof(u0)))" for u0 in ([1.0],)
-            sol = benchmark_nlsolve_iip(quadratic_f!, u0; solver = SimpleHouseholder{order}())
+            sol = benchmark_nlsolve_iip(
+                quadratic_f!, u0; solver = SimpleHouseholder{order}())
             @test SciMLBase.successful_retcode(sol)
             @test all(abs.(sol.u .* sol.u .- 2) .< 1e-9)
         end
@@ -112,7 +113,8 @@ end
         u0 in (1.0, [1.0], @SVector[1.0])
 
         probN = NonlinearProblem(quadratic_f, u0, 2.0)
-        @test all(solve(probN, SimpleHouseholder{2}(); termination_condition).u .≈ sqrt(2.0))
+        @test all(solve(probN, SimpleHouseholder{2}(); termination_condition).u .≈
+                  sqrt(2.0))
     end
 end
 

From 3936d006a424a6e3870bee3cf55f97df6cc09566 Mon Sep 17 00:00:00 2001
From: CompatHelper Julia <compathelper_noreply@julialang.org>
Date: Tue, 29 Oct 2024 19:05:22 +0000
Subject: [PATCH 374/375] CompatHelper: bump compat for TaylorDiff in
 [weakdeps] to 0.3, (keep existing compat)

---
 lib/SimpleNonlinearSolve/Project.toml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
index e7f4f625c..5d9d184a3 100644
--- a/lib/SimpleNonlinearSolve/Project.toml
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -24,16 +24,16 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 [weakdeps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
+TaylorDiff = "b36ab563-344f-407b-a36a-4f200bebf99c"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
-TaylorDiff = "b36ab563-344f-407b-a36a-4f200bebf99c"
 
 [extensions]
 SimpleNonlinearSolveChainRulesCoreExt = "ChainRulesCore"
 SimpleNonlinearSolveReverseDiffExt = "ReverseDiff"
+SimpleNonlinearSolveTaylorDiffExt = "TaylorDiff"
 SimpleNonlinearSolveTrackerExt = "Tracker"
 SimpleNonlinearSolveZygoteExt = "Zygote"
-SimpleNonlinearSolveTaylorDiffExt = "TaylorDiff"
 
 [compat]
 ADTypes = "1.9"
@@ -65,7 +65,7 @@ SciMLBase = "2.37.0"
 Setfield = "1.1.1"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4.2"
-TaylorDiff = "0.2.5"
+TaylorDiff = "0.2.5, 0.3"
 Test = "1.10"
 Tracker = "0.2.33"
 Zygote = "0.6.69"
@@ -95,4 +95,4 @@ Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["AllocCheck", "Aqua", "DiffEqBase", "ExplicitImports", "FiniteDiff", "ForwardDiff", "Hwloc", "InteractiveUtils", "LinearAlgebra",  "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReTestItems", "Reexport", "ReverseDiff",  "StaticArrays", "TaylorDiff", "Test", "Tracker", "Zygote"]
+test = ["AllocCheck", "Aqua", "DiffEqBase", "ExplicitImports", "FiniteDiff", "ForwardDiff", "Hwloc", "InteractiveUtils", "LinearAlgebra", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReTestItems", "Reexport", "ReverseDiff", "StaticArrays", "TaylorDiff", "Test", "Tracker", "Zygote"]

From b3db1a2347a7d96d768ad225f864a81fb579d27f Mon Sep 17 00:00:00 2001
From: Avik Pal <avikpal@mit.edu>
Date: Thu, 31 Oct 2024 15:30:35 -0400
Subject: [PATCH 375/375] docs: add note to readme

---
 lib/SimpleNonlinearSolve/README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
index 3ef4868bb..bbf80f250 100644
--- a/lib/SimpleNonlinearSolve/README.md
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -1,5 +1,8 @@
 # SimpleNonlinearSolve.jl
 
+> [!WARNING]
+> This package has been moved into a subpackage in NonlinearSolve (https://github.com/SciML/NonlinearSolve.jl/tree/master/lib/SimpleNonlinearSolve). Direct all questions and issues to NonlinearSolve.jl repository
+
 [![Join the chat at https://julialang.zulipchat.com #sciml-bridged](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/279055-sciml-bridged)
 [![Global Docs](https://img.shields.io/badge/docs-SciML-blue.svg)](https://docs.sciml.ai/NonlinearSolve/stable/)