Repository: FluxML/Optimisers.jl
Pull request #39 (2022-01-29 04:34:45)

Add some tests using StaticArrays
Testing with StaticArrays exposes some bugs (?) in how this handles element type:
* ~~Some `OptimiserChain` combinations promote to Float64.~~ when they have explicit Float64 parameters
* Most optimisers promote Float16, as their default parameters are Float32.

The PR also divides the tests into two files, one of which has loops over all optimisers, the other tries to test that the framework is doing what it should do.

Many of the optimiser convergence tests simply mark broken all that fail to converge. It's possible that some of these are real bugs. I'm surprised that minimising `sum(abs2, x + x')` isn't easy for all of them.

codecov-commenter: # [Codecov](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux) Report
> Merging [#39](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux) (b0eaa29) into [master](https://codecov.io/gh/FluxML/Optimisers.jl/commit/be3c943805d2bbefd9358dc626a697ec1b0b0351?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux) (be3c943) will **not change** coverage.
> The diff coverage is `100.00%`.

[![Impacted file tree graph](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39/graphs/tree.svg?width=650&height=150&src=pr&token=0pZyimon2U&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux)](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux)

```diff
@@           Coverage Diff           @@
##           master      #39   +/-   ##
=======================================
  Coverage   96.26%   96.26%           
=======================================
  Files           2        2           
  Lines         214      214           
=======================================
  Hits          206      206           
  Misses          8        8           
```


| [Impacted Files](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux) | Coverage Δ | |
|---|---|---|
| [src/rules.jl](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux#diff-c3JjL3J1bGVzLmps) | `99.36% <100.00%> (-0.64%)` | :arrow_down: |
| [src/interface.jl](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux#diff-c3JjL2ludGVyZmFjZS5qbA==) | `87.71% <0.00%> (+1.75%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux). Last update [be3c943...b0eaa29](https://codecov.io/gh/FluxML/Optimisers.jl/pull/39?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Flux).


Patch:
diff --git a/Project.toml b/Project.toml
index a688310..aafc7df 100644
--- a/Project.toml
+++ b/Project.toml
@@ -15,7 +15,9 @@ julia = "1.6"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["Test", "Zygote"]
+test = ["Test", "ChainRulesCore", "StaticArrays", "Zygote"]
diff --git a/src/rules.jl b/src/rules.jl
index 6123e94..6605dc5 100644
--- a/src/rules.jl
+++ b/src/rules.jl
@@ -16,7 +16,7 @@ Descent() = Descent(1f-1)
 init(o::Descent, x::AbstractArray) = nothing
 
 function apply!(o::Descent, state, x, dx)
-  η = convert(float(eltype(dx)), o.eta)
+  η = convert(float(eltype(x)), o.eta)
   
   return state, @.. dx * η
 end
@@ -477,7 +477,7 @@ ClipGrad() = ClipGrad(10f0)
 init(o::ClipGrad, x::AbstractArray) = nothing
 
 function apply!(o::ClipGrad, state, x, dx)
-  δ = convert(float(eltype(dx)), o.delta)
+  δ = convert(float(eltype(x)), o.delta)
   dx′ = @.. clamp(dx, -δ, δ)
 
   return state, dx′


Test patch:
diff --git a/test/rules.jl b/test/rules.jl
new file mode 100644
index 0000000..c950aeb
--- /dev/null
+++ b/test/rules.jl
@@ -0,0 +1,156 @@
+using Optimisers
+using ChainRulesCore, Functors, StaticArrays, Zygote
+using LinearAlgebra, Statistics, Test, Random
+
+Random.seed!(1)
+
+RULES = [
+  # All the rules at default settings
+  Descent(), ADAM(), Momentum(), Nesterov(), RMSProp(),
+  ADAGrad(), AdaMax(), ADADelta(), AMSGrad(), NADAM(),
+  ADAMW(), RADAM(), OADAM(), AdaBelief(),
+  # A few chained combinations:
+  OptimiserChain(WeightDecay(), ADAM(0.001)),
+  OptimiserChain(ClipNorm(), ADAM(0.001)),
+  OptimiserChain(ClipGrad(0.5), Momentum()),
+]
+
+name(o) = typeof(o).name.name
+name(o::OptimiserChain) = join(name.(o.opts), " → ")
+
+@testset "independence" begin
+  @testset "$(name(o))" for o in RULES
+    w = randn(10, 10)
+    w′ = randn(10, 10)
+    iloss(x, w, w′) = mean((w*x .- w′*x) .^ 2)
+    @test iloss(rand(10, 10), w, w′) > 1
+    st = Optimisers.setup(o, w)
+    for t = 1:10^5
+      x = rand(10)
+      gs = gradient(w -> iloss(x, w, w′), w)
+      st, w = Optimisers.update!(st, w, gs...)
+    end
+    @test iloss(rand(10, 10), w, w′) < 0.01
+  end
+end
+
+@testset verbose=true "simple sum" begin
+  @testset "$(name(o))" for o in RULES
+    m = shuffle!(reshape(1:64, 8, 8) .+ 0.0)
+    s = Optimisers.setup(o, m)
+    for _ in 1:10^5
+      g = gradient(x -> sum(abs2, x + x'), m)[1]
+      s, m = Optimisers.update!(s, m, g)
+    end
+    # @test sum(m) < sum(1:64)
+    if sum(m) < 1
+      @test sum(m) < 1
+    else
+      @show name(o) sum(m)/sum(1:64)
+      @test_broken sum(m) < 1
+    end
+  end
+end
+
+@testset "original" begin
+  @testset "$(name(o))" for o in RULES
+    w′ = (α = rand(3, 3), β = rand(3, 3))
+    w = (α = 5rand(3, 3), β = rand(3, 3))
+    st = Optimisers.setup(o, w)
+    loss(x, y) = mean((x.α .* x.β .- y.α .* y.β) .^ 2)
+    @test loss(w, w′) > 1
+    for i = 1:10^4
+      gs = gradient(x -> loss(x, w′), w)
+      st, w = Optimisers.update(st, w, gs...)
+    end
+    lw = loss(w, w′)
+    if o isa ADADelta
+      @show name(o) loss(w, w′)
+      @test_broken lw < 0.001
+    else
+      @test lw < 0.001
+    end
+  end
+end
+
+@testset verbose=true "StaticArrays" begin
+  @testset "$(name(o))" for o in RULES
+    W1 = @SMatrix randn(10, 10)
+    b1 = @SVector randn(10)
+    W2 = @SMatrix randn(10, 10)
+    model = (; W1, b1, W2, tanh)
+    s_loss(m, x, y) = sum(abs2, m.W2 * (m.tanh).(m.W1*x .+ m.b1) .- y)
+    # x = @SMatrix randn(10, 10)
+    # y = @SMatrix randn(10, 10)  # gives an error from sum(; dims=())
+    x = @SVector randn(10)
+    y = @SVector randn(10)
+    @test s_loss(model, x, y) > 10
+    state = Optimisers.setup(o, model)
+    for t = 1:10^3
+      g = gradient(m -> s_loss(m, x, y), model)[1]
+      state, model = Optimisers.update!(state, model, g)
+    end
+    if o isa Union{Descent, RMSProp, ADAGrad, ADADelta, NADAM}
+      @show name(o) s_loss(model, x, y)
+      @test_broken s_loss(model, x, y) < 1
+    else
+      @test s_loss(model, x, y) < 1
+    end
+  end
+end
+
+@testset verbose=true "element types" begin
+  @testset "$(name(o))" for o in RULES
+    marray = (Float32[1,2], Float64[3,4], Float16[5,6])
+    types = map(eltype, marray)
+
+    # This is a weak test, as it copies & then does `update!`
+    uparray = Optimisers.update(Optimisers.setup(o, marray), marray, marray)[2]
+    @test map(eltype, uparray) == types
+
+    # Static version is truly out-of-place:
+    mstatic = (SA{Float32}[1,2], SA{Float64}[3,4]) # , SA{Float16}[5,6])  with Float16, all fail
+    upstatic = Optimisers.update(Optimisers.setup(o, mstatic), mstatic, mstatic)[2]
+    if o isa OptimiserChain && o.opts[2] isa ADAM  # These promote to Float64
+      @test_broken map(eltype, upstatic) == types[1:2]
+    else
+      @test map(eltype, upstatic) == types[1:2]
+    end
+    @test upstatic[1] isa SVector
+
+    # With ordinary Array gradient, what happens?
+    upstatic2 = Optimisers.update(Optimisers.setup(o, mstatic), mstatic, marray[1:2])[2]
+    # @test map(eltype, upstatic2) == types[1:2]  # same information
+    if upstatic2[1] isa SVector
+      @test upstatic2[1] isa SVector
+    else
+      @test_broken upstatic2[1] isa SVector
+    end
+  end
+end
+
+@testset "gradient types" begin
+  @testset "$(name(o))" for o in RULES
+    x = (a = ones(2,2), b = transpose(ones(2,2)))
+    s = Optimisers.setup(o, x)
+
+    _, x1 = Optimisers.update(s, x, (a = [1 2; 3 4], b = nothing))
+    @test x1.a != ones(2,2)
+    @test x1.b == ones(2,2)
+
+    _, xfill = Optimisers.update(s, x, (a = Zygote.Fill(2.0,2,2), b = Zygote.Fill(true,2,2)))
+    @test xfill.a != ones(2,2)
+    @test xfill.b != ones(2,2)
+
+    bc = Optimisers.@.. 1 + log([2 3; 4 5]) / 6
+    _, xbc = Optimisers.update(s, x, (a = bc, b = bc))
+    @test xbc.a != ones(2,2)
+    @test xbc.b != ones(2,2)
+
+    th = ChainRulesCore.@thunk @. 1 + log([2 3; 4 5]) / 6
+    _, xth = Optimisers.update(s, x, (a = bc, b = bc))
+    @test xth.a != ones(2,2)
+    @test xth.b != ones(2,2)
+  end
+end
+
diff --git a/test/runtests.jl b/test/runtests.jl
index 0ed1bbd..8233ef7 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,4 +1,5 @@
-using Optimisers, Functors, Zygote
+using Optimisers
+using ChainRulesCore, Functors, StaticArrays, Zygote
 using LinearAlgebra, Statistics, Test, Random
 using Optimisers: @..
 
@@ -13,152 +14,114 @@ Functors.@functor TwoThirds (a, c)
 Optimisers.trainable(x::TwoThirds) = (a = x.a,)
 
 @testset verbose=true "Optimisers.jl" begin
-
-  @testset "very basics" begin
-    m = ([1.0, 2.0],)
-    mid = objectid(m[1])
-    g = ([25, 33],)
-    o = Descent(0.1)
-    s = Optimisers.setup(o, m)
-    
-    s2, m2 = Optimisers.update(s, m, g)
-    @test m[1] == 1:2  # not mutated
-    @test Optimisers.iswriteable(m[1])
-    @test m2[1] ≈ [1,2] .- 0.1 .* [25, 33]
-
-    s3, m3 = Optimisers.update!(s, m, g)
-    @test objectid(m3[1]) == mid
-    @test m3[1] ≈ [1,2] .- 0.1 .* [25, 33]
-  end
-
-  @testset "rule: $(first(string(o), 42))" for o in (
-                     Descent(), ADAM(), Momentum(), Nesterov(), RMSProp(),
-                     ADAGrad(), AdaMax(), ADADelta(), AMSGrad(), NADAM(),
-                     ADAMW(), RADAM(), OADAM(), AdaBelief()
-                     )
-    w′ = (α = rand(3, 3), β = rand(3, 3))
-
-    # Original example
-    w = (α = 5rand(3, 3), β = rand(3, 3))
-    st = Optimisers.setup(o, w)
-    loss(x, y) = mean((x.α .* x.β .- y.α .* y.β) .^ 2)
-    @test loss(w, w′) > 1
-    for i = 1:10^4
-      gs = gradient(x -> loss(x, w′), w)
-      st, w = Optimisers.update(st, w, gs...)
-    end
-    lw = loss(w, w′)
-    if o isa ADADelta
-      @test_broken lw < 0.001
-    else
-      @test lw < 0.001
+  @testset verbose=true "Features" begin
+
+    @testset "very basics" begin
+      m = ([1.0, 2.0],)
+      mid = objectid(m[1])
+      g = ([25, 33],)
+      o = Descent(0.1)
+      s = Optimisers.setup(o, m)
+      
+      s2, m2 = Optimisers.update(s, m, g)
+      @test m[1] == 1:2  # not mutated
+      @test Optimisers.iswriteable(m[1])
+      @test m2[1] ≈ [1,2] .- 0.1 .* [25, 33]
+
+      s3, m3 = Optimisers.update!(s, m, g)
+      @test objectid(m3[1]) == mid
+      @test m3[1] ≈ [1,2] .- 0.1 .* [25, 33]
     end
 
-    # Slightly harder variant
-    m = (α = randn(3), β = transpose(5rand(3,3)), γ = (rand(2), tanh))  # issue 28
-    st = Optimisers.setup(o, m)
-    @test loss(m, w′) > 1
-    for i = 1:10^4
-      gs = gradient(x -> loss(x, w′), m)
-      st, m = Optimisers.update!(st, m, gs...)
-    end
-    lm = loss(m, w′)
-    if lm < 0.1
-      @test lm < 0.1
-    else
-      @test_broken lm < 0.1  # @test keyword broken doesn't exist on Julia 1.6
+    @testset "gradient clipping" begin
+      m = (α = ([0], sin), γ = rand(3))
+      s1 = Optimisers.setup(ClipGrad(13), m)
+      _, m1 = Optimisers.update(s1, m, (α = nothing, γ = [1,10,100],))
+      @test m.γ .- m1.γ ≈ [1, 10, 13]
+
+      s2 = Optimisers.setup(ClipNorm(10), m)
+      _, m2 = Optimisers.update(s2, m, (α = ([0.1], nothing), γ = [1,10,100],))
+      @test only(m.α[1] .- m2.α[1]) ≈ 0.1
+      @test norm(m.γ .- m2.γ) ≈ 10
+      @test_throws DomainError Optimisers.update(s2, m, (α = [0.1], γ = [1,10,NaN],))
+
+      s3 = Optimisers.setup(ClipNorm(5, 1; throw=false), m)
+      _, m3 = Optimisers.update(s3, m, (α = ([0.1], nothing), γ = [1,10,100],))
+      @test only(m.α[1] .- m3.α[1]) ≈ 0.1
+      @test norm(m.γ .- m3.γ, 1) ≈ 5
+      _, m3n = Optimisers.update!(s3, m, (α = nothing, γ = [1,10,Inf],))
+      @test isnan(m3n.γ[3])
     end
 
-  end
+    @testset "OptimiserChain" begin
+      x = [1,10,100]; dx = [1,2,3];
+      @test Optimisers.update(Optimisers.setup(WeightDecay(0.1), x), x, dx)[2] ≈ [1-0.1-1, 10-1-2, 100-10-3]
+      @test Optimisers.update(Optimisers.setup(ClipGrad(2), x), x, dx)[2] ≈ [1-1, 10-2, 100-2]
 
-  @testset "OptimiserChain with $pre" for pre in (WeightDecay(), ClipGrad(), ClipNorm())
-    Random.seed!(84)
-    w = randn(10, 10)
-    w′ = randn(10, 10)
-    loss(x, w, w′) = mean((w*x .- w′*x) .^ 2)
-    @test loss(rand(10, 10), w, w′) > 1
-    opt = OptimiserChain(pre, ADAM(0.001))
-    st = Optimisers.init(opt, w)
-    for t = 1:10^5
-      x = rand(10)
-      gs = gradient(w -> loss(x, w, w′), w)
-      st, w = Optimisers.update!(st, w, gs...)
+      o2 = OptimiserChain(ClipGrad(2), WeightDecay(0.1))
+      @test Optimisers.update(Optimisers.setup(o2, x), x, dx)[2] ≈ [1-0.1-1, 10-1-2, 100-10-2]
+
+      o2r = OptimiserChain(WeightDecay(0.1), ClipGrad(2))
+      @test Optimisers.update(Optimisers.setup(o2r, x), x, dx)[2] != [1-0.1-1, 10-2, 100-2]
     end
-    @test_broken loss(rand(10, 10), w, w′) < 0.01
-  end
 
-  @testset "gradient clipping" begin
-    m = (α = ([0], sin), γ = rand(3))
-    s1 = Optimisers.setup(ClipGrad(13), m)
-    _, m1 = Optimisers.update(s1, m, (α = nothing, γ = [1,10,100],))
-    @test m.γ .- m1.γ ≈ [1, 10, 13]
-
-    s2 = Optimisers.setup(ClipNorm(10), m)
-    _, m2 = Optimisers.update(s2, m, (α = ([0.1], nothing), γ = [1,10,100],))
-    @test only(m.α[1] .- m2.α[1]) ≈ 0.1
-    @test norm(m.γ .- m2.γ) ≈ 10
-    @test_throws DomainError Optimisers.update(s2, m, (α = [0.1], γ = [1,10,NaN],))
-
-    s3 = Optimisers.setup(ClipNorm(5, 1; throw=false), m)
-    _, m3 = Optimisers.update(s3, m, (α = ([0.1], nothing), γ = [1,10,100],))
-    @test only(m.α[1] .- m3.α[1]) ≈ 0.1
-    @test norm(m.γ .- m3.γ, 1) ≈ 5
-    _, m3n = Optimisers.update!(s3, m, (α = nothing, γ = [1,10,Inf],))
-    @test isnan(m3n.γ[3])
-  end
+    @testset "trainable subset" begin
+      # Foo has an old-style tuple trainable, both elements
+      mf = Foo([1,2], (a = sin, b = [3,4], c = 5))
+      sf = Optimisers.setup(Descent(0.1), mf)
+      gf = (x = nothing, y = (a = nothing, b = [1,1], c = 1))
+      _, mf2 = Optimisers.update(sf, mf, gf)
+      @test mf2.x == [1,2]
+      @test mf2.y == (a = sin, b = [2.9, 3.9], c = 5)
+
+      # TwoThirds has functor a,c only, and trainable a only
+      mt = TwoThirds(Float32[1,2], Float32[3,4], Float32[5,6])
+      mt10 = fmap(x -> 10x, mt)
+      @test mt10.a == [10, 20]
+      @test mt10.b == [3, 4]
+      @test mt10.c == [50, 60]
+      st = Optimisers.setup(Momentum(0.1, 0.9), mt)
+      gt = gradient(m -> sum(abs2, m.a) + 100sum(abs2, m.b), mt)
+      _, mtup = Optimisers.update(st, mt, gt...)
+      @test mtup.a ≈ [0.8, 1.6]
+      @test mtup.b == [3, 4]
+      @test mtup.c == [5, 6]
+
+      # Various kinds of missing branches together:
+      m = Foo(
+          TwoThirds(Foo(1.0, Float32[2,3,4]), 5.0, Float32[6,7]),
+          TwoThirds((p = Float32[1,2,3],), sin, (q = 4.0, r = cos,)),
+          )
+      s = Optimisers.setup(Momentum(0.1, 0.9), m)
+      g = gradient(m -> sum(abs2, m.x.a.y) + m.x.b^2 + log(m.y.c.q), m)
+      @test Optimisers.update!(s, m, g...)[2] isa Foo
+    end
 
-  @testset "trainable subset" begin
-    # Foo has an old-style tuple trainable, both elements
-    mf = Foo([1,2], (a = sin, b = [3,4], c = 5))
-    sf = Optimisers.setup(Descent(0.1), mf)
-    gf = (x = nothing, y = (a = nothing, b = [1,1], c = 1))
-    _, mf2 = Optimisers.update(sf, mf, gf)
-    @test mf2.x == [1,2]
-    @test mf2.y == (a = sin, b = [2.9, 3.9], c = 5)
-
-    # TwoThirds has functor a,c only, and trainable a only
-    mt = TwoThirds(Float32[1,2], Float32[3,4], Float32[5,6])
-    mt10 = fmap(x -> 10x, mt)
-    @test mt10.a == [10, 20]
-    @test mt10.b == [3, 4]
-    @test mt10.c == [50, 60]
-    st = Optimisers.setup(Momentum(0.1, 0.9), mt)
-    gt = gradient(m -> sum(abs2, m.a) + 100sum(abs2, m.b), mt)
-    _, mtup = Optimisers.update(st, mt, gt...)
-    @test mtup.a ≈ [0.8, 1.6]
-    @test mtup.b == [3, 4]
-    @test mtup.c == [5, 6]
-
-    # Various kinds of missing branches together:
-    m = Foo(
-        TwoThirds(Foo(1.0, Float32[2,3,4]), 5.0, Float32[6,7]),
-        TwoThirds((p = Float32[1,2,3],), sin, (q = 4.0, r = cos,)),
-        )
-    s = Optimisers.setup(Momentum(0.1, 0.9), m)
-    g = gradient(m -> sum(abs2, m.x.a.y) + m.x.b^2 + log(m.y.c.q), m)
-    @test Optimisers.update!(s, m, g...)[2] isa Foo
-  end
+    @testset "broadcasting macro" begin
+      x = [1.0, 2.0]; y = [3,4]; z = [5,6]
+      @test (@.. x + y * z) isa Broadcast.Broadcasted
+      bc = @.. x + y * z
+      @test (y .+ 2 .* bc) == [35,56]
+
+      @test (@.. x = y * z) isa Array
+      @test x == y .* z  # mutated
+      r = 1.0:2.0  # immutable
+      @test (@.. r = y * z) isa Array
+      @test (@.. r = y * z) == y .* z
+    end
 
-  @testset "broadcasting macro" begin
-    x = [1.0, 2.0]; y = [3,4]; z = [5,6]
-    @test (@.. x + y * z) isa Broadcast.Broadcasted
-    bc = @.. x + y * z
-    @test (y .+ 2 .* bc) == [35,56]
-
-    @test (@.. x = y * z) isa Array
-    @test x == y .* z  # mutated
-    r = 1.0:2.0  # immutable
-    @test (@.. r = y * z) isa Array
-    @test (@.. r = y * z) == y .* z
-  end
+    @testset "tied weights" begin
+      ok = (1.0:3.0, sin, "abc", :abc)
+      m = (α = ok, β = rand(3), γ = ok)
+      m1 = (rand(3), m, rand(3))
+      @test Optimisers.setup(ADAMW(), m1) isa Tuple
+      m2 = (rand(3), m, rand(3), m, rand(3))  # illegal
+      @test_throws ArgumentError Optimisers.setup(ADAMW(), m2)
+    end
 
-  @testset "tied weights" begin
-    ok = (1.0:3.0, sin, "abc", :abc)
-    m = (α = ok, β = rand(3), γ = ok)
-    m1 = (rand(3), m, rand(3))
-    @test Optimisers.setup(ADAMW(), m1) isa Tuple
-    m2 = (rand(3), m, rand(3), m, rand(3))  # illegal
-    @test_throws ArgumentError Optimisers.setup(ADAMW(), m2)
+    @info "finished feature testing"
+  end
+  @testset verbose=true "Optimisation Rules" begin
+    include("rules.jl")
   end
-
 end