Repository: JuliaML/TableTransforms.jl (julia)

Issue:
Add `StdFeats` transform to standardize features
The `StdFeats` will apply the `ZScore` transform the the `Continuous` columns of a table. In the future it might do something else with other scientific types.

Pull request:
closes #233

Patch:
diff --git a/docs/src/transforms.md b/docs/src/transforms.md
index 220ccccc..797db6ad 100644
--- a/docs/src/transforms.md
+++ b/docs/src/transforms.md
@@ -44,6 +44,12 @@ Rename
 StdNames
 ```
 
+## StdFeats
+
+```@docs
+StdFeats
+```
+
 ## Sort
 
 ```@docs
diff --git a/src/TableTransforms.jl b/src/TableTransforms.jl
index 2e7bc226..5ea3bcb1 100644
--- a/src/TableTransforms.jl
+++ b/src/TableTransforms.jl
@@ -56,6 +56,7 @@ export
   Except,
   Rename,
   StdNames,
+  StdFeats,
   Sort,
   Sample,
   Filter,
diff --git a/src/transforms.jl b/src/transforms.jl
index 685e1796..8d47e31c 100644
--- a/src/transforms.jl
+++ b/src/transforms.jl
@@ -264,10 +264,12 @@ end
 # IMPLEMENTATIONS
 # ----------------
 
+include("transforms/utils.jl")
 include("transforms/select.jl")
 include("transforms/satisfies.jl")
 include("transforms/rename.jl")
 include("transforms/stdnames.jl")
+include("transforms/stdfeats.jl")
 include("transforms/sort.jl")
 include("transforms/sample.jl")
 include("transforms/filter.jl")
diff --git a/src/transforms/stdfeats.jl b/src/transforms/stdfeats.jl
new file mode 100644
index 00000000..319adca6
--- /dev/null
+++ b/src/transforms/stdfeats.jl
@@ -0,0 +1,58 @@
+# ------------------------------------------------------------------
+# Licensed under the MIT License. See LICENSE in the project root.
+# ------------------------------------------------------------------
+
+"""
+    StdFeats()
+
+Standardizes the columns of the table based on scientific types:
+
+* `Continuous`: `ZScore`
+* `Categorical`: `Identity`
+* `Unknown`: `Identity`
+"""
+struct StdFeats <: StatelessFeatureTransform end
+
+isrevertible(::Type{StdFeats}) = true
+
+_stdfun(x) = _stdfun(elscitype(x), x)
+_stdfun(::Type, x) = identity, identity
+function _stdfun(::Type{Continuous}, x)
+  μ = mean(x)
+  σ = std(x, mean=μ)
+  stdfun = x -> zscore(x, μ, σ)
+  revfun = y -> revzscore(y, μ, σ)
+  stdfun, revfun
+end
+
+function applyfeat(::StdFeats, feat, prep)
+  cols = Tables.columns(feat)
+  names = Tables.columnnames(cols)
+
+  tuples = map(names) do name
+    x = Tables.getcolumn(cols, name)
+    stdfun, revfun = _stdfun(x)
+    stdfun(x), revfun
+  end
+
+  columns = first.(tuples)
+  fcache = last.(tuples)
+
+  𝒯 = (; zip(names, columns)...)
+  newfeat = 𝒯 |> Tables.materializer(feat)
+
+  newfeat, fcache
+end
+
+function revertfeat(::StdFeats, newfeat, fcache)
+  cols = Tables.columns(newfeat)
+  names = Tables.columnnames(cols)
+
+  columns = map(names, fcache) do name, revfun
+    y = Tables.getcolumn(cols, name)
+    revfun(y)
+  end
+
+  𝒯 = (; zip(names, columns)...)
+  𝒯 |> Tables.materializer(newfeat)
+end
diff --git a/src/transforms/utils.jl b/src/transforms/utils.jl
new file mode 100644
index 00000000..185fdee5
--- /dev/null
+++ b/src/transforms/utils.jl
@@ -0,0 +1,7 @@
+# ------------------------------------------------------------------
+# Licensed under the MIT License. See LICENSE in the project root.
+# ------------------------------------------------------------------
+
+zscore(x, μ, σ) = @. (x - μ) / σ
+
+revzscore(y, μ, σ) = @. σ * y + μ
diff --git a/src/transforms/zscore.jl b/src/transforms/zscore.jl
index b8069f62..1bec2e76 100644
--- a/src/transforms/zscore.jl
+++ b/src/transforms/zscore.jl
@@ -46,6 +46,6 @@ function colcache(::ZScore, x)
   (μ=μ, σ=σ)
 end
 
-colapply(::ZScore, x, c) = @. (x - c.μ) / c.σ
+colapply(::ZScore, x, c) = zscore(x, c.μ, c.σ)
 
-colrevert(::ZScore, y, c) = @. c.σ * y + c.μ
+colrevert(::ZScore, y, c) = revzscore(y, c.μ, c.σ)


Test patch:
diff --git a/test/shows.jl b/test/shows.jl
index 9c5fcff5..d45ce279 100644
--- a/test/shows.jl
+++ b/test/shows.jl
@@ -85,6 +85,18 @@
     └─ spec = :upperflat"""
   end
 
+  @testset "StdFeats" begin
+    T = StdFeats()
+
+    # compact mode
+    iostr = sprint(show, T)
+    @test iostr == "StdFeats()"
+
+    # full mode
+    iostr = sprint(show, MIME("text/plain"), T)
+    @test iostr == "StdFeats transform"
+  end
+
   @testset "Sort" begin
     T = Sort([:a, :c], rev=true)
 
diff --git a/test/transforms.jl b/test/transforms.jl
index 25787516..8a389ccd 100644
--- a/test/transforms.jl
+++ b/test/transforms.jl
@@ -3,6 +3,7 @@ transformfiles = [
   "rename.jl",
   "satisfies.jl",
   "stdnames.jl",
+  "stdfeats.jl",
   "sort.jl",
   "sample.jl",
   "filter.jl",
diff --git a/test/transforms/stdfeats.jl b/test/transforms/stdfeats.jl
new file mode 100644
index 00000000..e60df22b
--- /dev/null
+++ b/test/transforms/stdfeats.jl
@@ -0,0 +1,26 @@
+@testset "StdFeats" begin
+  @test isrevertible(StdFeats())
+
+  a = rand(1:10, 100)
+  b = rand(Normal(7, 10), 100)
+  c = rand('a':'z', 100)
+  d = rand(Normal(15, 2), 100)
+  e = rand(["y", "n"], 100)
+  t = Table(; a, b, c, d, e)
+
+  T = StdFeats()
+  n, c = apply(T, t)
+  @test n.a == t.a
+  @test isapprox(mean(n.b), 0; atol=1e-6)
+  @test isapprox(std(n.b), 1; atol=1e-6)
+  @test n.c == t.c
+  @test isapprox(mean(n.d), 0; atol=1e-6)
+  @test isapprox(std(n.d), 1; atol=1e-6)
+  @test n.e == t.e
+  tₒ = revert(T, n, c)
+  @test tₒ.a == t.a
+  @test tₒ.b ≈ t.b
+  @test tₒ.c == t.c
+  @test tₒ.d ≈ t.d
+  @test tₒ.e == t.e
+end