Repository: JuliaML/TableTransforms.jl (julia)

Issue:
Unexpected behavior for row table
I expected this:

```julia
julia> X = Tables.table(rand(10, 3))
Tables.MatrixTable{Matrix{Float64}} with 10 rows, 3 columns, and schema:
 :Column1  Float64
 :Column2  Float64
 :Column3  Float64

julia> rt = Tables.rowtable(X)
10-element Vector{NamedTuple{(:Column1, :Column2, :Column3), Tuple{Float64, Float64, Float64}}}:
 (Column1 = 0.5565428815102202, Column2 = 0.9460318649319521, Column3 = 0.3359377652605777)
 (Column1 = 0.5075982188653294, Column2 = 0.18315493443083453, Column3 = 0.9682078694091394)
 (Column1 = 0.9963268867042159, Column2 = 0.9329899214178856, Column3 = 0.5470116730645322)
 (Column1 = 0.3586863538486338, Column2 = 0.028403436910423796, Column3 = 0.37539265155858015)
 (Column1 = 0.6219233944778164, Column2 = 0.3499360310552866, Column3 = 0.14038853850080102)
 (Column1 = 0.587164895366161, Column2 = 0.9365631957076301, Column3 = 0.4638888590010579)
 (Column1 = 0.5896922517858991, Column2 = 0.29876303863960274, Column3 = 0.8361667225744209)
 (Column1 = 0.3924751962812454, Column2 = 0.7981085942387889, Column3 = 0.11151780408924594)
 (Column1 = 0.26805907763502645, Column2 = 0.6863527449570381, Column3 = 0.5322558300965325)
 (Column1 = 0.4000480227789831, Column2 = 0.5888482501962962, Column3 = 0.25538366367233634)

julia> X |> Center()
(Column1 = [0.02869116358486712, -0.020253499060023716, 0.46847516877886275, -0.16916536407671934, 0.09407167655246329, 0.059313177440807885, 0.061840533860546, -0.13537652164410774, -0.25979264029032667, -0.12780369514637002], Column2 = [0.37111666368337826, -0.39176026681773934, 0.35807472016931174, -0.5465117643381501, -0.22497917019328728, 0.36164799445905627, -0.27615216260897113, 0.223193392990215, 0.11143754370846426, 0.013933048947722293], Column3 = [-0.12067737246214477, 0.5115927316864168, 0.09039653534180975, -0.08122248616414235, -0.3162265992219215, 0.007273721278335421, 0.37955158485169843, -0.34509733363347656, 0.07564069237381005, -0.20123147405038616])
```

But not this:

```julia
julia> rt |> Center()
NamedTuple{(), Tuple{}}[]
```

Pull request:
closes #61

Patch:
diff --git a/src/colspec.jl b/src/colspec.jl
index bbd40caa..9c81dc9a 100644
--- a/src/colspec.jl
+++ b/src/colspec.jl
@@ -24,7 +24,8 @@ end
 2. use `choose(colspec, names)` function in apply:
 ```julia
 function apply(transform::MyTransform, table)
-  names = Tables.columnnames(table)
+  cols = Tables.columns(table)
+  names = Tables.columnnames(cols)
   # selected column names
   snames = choose(transform.colspec, names)
   # code...
diff --git a/src/transforms.jl b/src/transforms.jl
index 15e45b89..80eb7e19 100644
--- a/src/transforms.jl
+++ b/src/transforms.jl
@@ -130,8 +130,8 @@ function apply(transform::Colwise, table)
   end
 
   # retrieve column names and values
-  names = Tables.columnnames(table)
   cols  = Tables.columns(table)
+  names = Tables.columnnames(cols)
 
   # function to transform a single column
   function colfunc(n)
@@ -159,9 +159,9 @@ function revert(transform::Colwise, newtable, cache)
   @assert isrevertible(transform) "transform is not revertible"
 
   # transformed columns
-  names = Tables.columnnames(newtable)
   cols  = Tables.columns(newtable)
-
+  names = Tables.columnnames(cols)
+  
   # function to transform a single column
   function colfunc(i)
     n = names[i]
@@ -185,9 +185,9 @@ function reapply(transform::Colwise, table, cache)
   end
 
   # retrieve column names and values
-  names = Tables.columnnames(table)
   cols  = Tables.columns(table)
-
+  names = Tables.columnnames(cols)
+  
   # check that cache is valid
   @assert length(names) == length(cache) "invalid cache for table"
 
diff --git a/src/transforms/coerce.jl b/src/transforms/coerce.jl
index 4b847609..5f7455d3 100644
--- a/src/transforms/coerce.jl
+++ b/src/transforms/coerce.jl
@@ -37,8 +37,9 @@ function apply(transform::Coerce, table)
 end
 
 function revert(transform::Coerce, newtable, cache)
-  names = Tables.columnnames(newtable)
   cols = Tables.columns(newtable)
+  names = Tables.columnnames(cols)
+  
   oldcols = map(zip(cache, names)) do (T, n)
     x = Tables.getcolumn(cols, n)
     collect(T, x)
@@ -47,4 +48,3 @@ function revert(transform::Coerce, newtable, cache)
   𝒯 = (; zip(names, oldcols)...)
   𝒯 |> Tables.materializer(newtable)
 end
-
diff --git a/src/transforms/eigenanalysis.jl b/src/transforms/eigenanalysis.jl
index 005918be..ed2ad7c5 100644
--- a/src/transforms/eigenanalysis.jl
+++ b/src/transforms/eigenanalysis.jl
@@ -53,7 +53,8 @@ function apply(transform::EigenAnalysis, table)
   end
 
   # original columns names
-  names = Tables.columnnames(table)
+  cols = Tables.columns(table)
+  names = Tables.columnnames(cols)
 
   # table as matrix
   X = Tables.matrix(table)
@@ -77,7 +78,8 @@ end
 
 function revert(::EigenAnalysis, newtable, cache)
   # transformed column names
-  names = Tables.columnnames(newtable)
+  cols = Tables.columns(newtable)
+  names = Tables.columnnames(cols)
 
   # table as matrix
   Z = Tables.matrix(newtable)
@@ -103,7 +105,8 @@ function reapply(transform::EigenAnalysis, table, cache)
   end
 
   # original columns names
-  names = Tables.columnnames(table)
+  cols = Tables.columns(table)
+  names = Tables.columnnames(cols)
 
   # table as matrix
   X = Tables.matrix(table)
diff --git a/src/transforms/filter.jl b/src/transforms/filter.jl
index 5a0f775d..9023bcb9 100644
--- a/src/transforms/filter.jl
+++ b/src/transforms/filter.jl
@@ -102,16 +102,17 @@ _nonmissing(::Type{Union{Missing,T}}, x) where {T} = collect(T, x)
 _nonmissing(x) = _nonmissing(eltype(x), x)
 
 function apply(transform::DropMissing, table)
-  names = Tables.columnnames(table)
+  cols = Tables.columns(table)
+  names = Tables.columnnames(cols)
   types = Tables.schema(table).types
   snames = choose(transform.colspec, names)
   ftrans = _ftrans(transform, snames)
   newtable, fcache = apply(ftrans, table)
 
   # post-processing
-  cols = Tables.columns(newtable)
+  ncols = Tables.columns(newtable)
   pcols = map(names) do n
-    x = Tables.getcolumn(cols, n)
+    x = Tables.getcolumn(ncols, n)
     n ∈ snames ? _nonmissing(x) : x
   end
   𝒯 = (; zip(names, pcols)...)
@@ -125,7 +126,7 @@ function revert(::DropMissing, newtable, cache)
 
   # pre-processing
   cols = Tables.columns(newtable)
-  names = Tables.columnnames(newtable)
+  names = Tables.columnnames(cols)
   pcols = map(zip(types, names)) do (T, n)
     x = Tables.getcolumn(cols, n)
     collect(T, x)
diff --git a/src/transforms/functional.jl b/src/transforms/functional.jl
index 2031711b..1c6ab4c9 100644
--- a/src/transforms/functional.jl
+++ b/src/transforms/functional.jl
@@ -59,7 +59,7 @@ _functuple(func::NamedTuple, names) = func
 
 function apply(transform::Functional, table) 
   cols = Tables.columns(table)
-  names = Tables.columnnames(table)
+  names = Tables.columnnames(cols)
   funcs = _functuple(transform.func, names)
   
   ncols = map(names) do nm
@@ -77,7 +77,7 @@ function revert(transform::Functional, newtable, cache)
   @assert isrevertible(transform) "Transform is not revertible."
 
   cols = Tables.columns(newtable)
-  names = Tables.columnnames(newtable)
+  names = Tables.columnnames(cols)
   funcs = _functuple(transform.func, names)
 
   ocols = map(names) do nm
diff --git a/src/transforms/parallel.jl b/src/transforms/parallel.jl
index 0fb684b3..ada90e33 100644
--- a/src/transforms/parallel.jl
+++ b/src/transforms/parallel.jl
@@ -33,7 +33,8 @@ function apply(p::Parallel, table)
   newtable = tablehcat(tables)
 
   # save original column names
-  onames = Tables.columnnames(table)
+  ocols = Tables.columns(table)
+  onames = Tables.columnnames(ocols)
 
   # find first revertible transform
   ind = findfirst(isrevertible, p.transforms)
@@ -42,7 +43,8 @@ function apply(p::Parallel, table)
   rinfo = if isnothing(ind)
     nothing
   else
-    tnames = Tables.columnnames.(tables)
+    tcols  = Tables.columns.(tables)
+    tnames = Tables.columnnames.(tcols)
     ncols  = length.(tnames)
     nrcols = ncols[ind]
     start  = sum(ncols[1:ind-1]) + 1
@@ -99,7 +101,7 @@ function tablehcat(tables)
   varsdict = Set{Symbol}()
   for 𝒯 in tables
     cols = Tables.columns(𝒯)
-    vars = Tables.columnnames(𝒯)
+    vars = Tables.columnnames(cols)
     vals = [Tables.getcolumn(cols, var) for var in vars]
     for (var, val) in zip(vars, vals)
       while var ∈ varsdict
diff --git a/src/transforms/rename.jl b/src/transforms/rename.jl
index df6e7192..d14d950a 100644
--- a/src/transforms/rename.jl
+++ b/src/transforms/rename.jl
@@ -35,7 +35,8 @@ end
 
 
 function _rename(names, table)
-  oldnames = Tables.columnnames(table)
+  cols = Tables.columns(table)
+  oldnames = Tables.columnnames(cols)
 
   # check if requested renames exist in the table
   @assert keys(names) ⊆ oldnames "invalid column names"
@@ -46,7 +47,6 @@ function _rename(names, table)
   end
 
   # table with new tables
-  cols = Tables.columns(table)
   vals = [Tables.getcolumn(cols, name) for name in oldnames]
   𝒯 = (; zip(newnames, vals)...) |> Tables.materializer(table)
   
diff --git a/src/transforms/select.jl b/src/transforms/select.jl
index c684d636..57483725 100644
--- a/src/transforms/select.jl
+++ b/src/transforms/select.jl
@@ -80,8 +80,11 @@ Select(cols::T...) where {T<:ColSelector} =
 isrevertible(::Type{<:Select}) = true
 
 function apply(transform::Select, table)
+  # original columns
+  cols = Tables.columns(table)
+
   # retrieve relevant column names
-  allcols = collect(Tables.columnnames(table))
+  allcols = collect(Tables.columnnames(cols))
   select  = choose(transform.colspec, allcols)
   reject  = setdiff(allcols, select)
 
@@ -95,9 +98,6 @@ function apply(transform::Select, table)
   reject = reject[sorted]
   rinds  = rinds[sorted]
 
-  # original columns
-  cols = Tables.columns(table)
-
   # rejected columns
   rcols = [Tables.getcolumn(cols, name) for name in reject]
 
@@ -107,7 +107,7 @@ end
 function revert(::Select, newtable, cache)
   # selected columns
   cols   = Tables.columns(newtable)
-  select = Tables.columnnames(newtable)
+  select = Tables.columnnames(cols)
   scols  = [Tables.getcolumn(cols, name) for name in select]
 
   # rejected columns
@@ -162,7 +162,8 @@ Reject(cols::T...) where {T<:ColSelector} =
 isrevertible(::Type{<:Reject}) = true
 
 function apply(transform::Reject, table)
-  allcols = Tables.columnnames(table)
+  cols = Tables.columns(table)
+  allcols = Tables.columnnames(cols)
   reject  = choose(transform.colspec, allcols)
   select  = setdiff(allcols, reject)
   strans  = Select(select)


Test patch:
diff --git a/test/transforms.jl b/test/transforms.jl
index ce7025ea..64f12ffb 100644
--- a/test/transforms.jl
+++ b/test/transforms.jl
@@ -123,6 +123,14 @@
     tₒ = revert(T, n, c)
     @test t == tₒ
 
+    # row table
+    rt = Tables.rowtable(t)
+    T = Select(r"y")
+    n, c = apply(T, rt)
+    @test Tables.columnnames(n) == [:y1, :y2]
+    rtₒ = revert(T, n, c)
+    @test rt == rtₒ
+
     # throws: Select without arguments
     @test_throws ArgumentError Select()
     @test_throws ArgumentError Select(())
@@ -263,6 +271,14 @@
     tₒ = revert(T, n, c)
     @test t == tₒ
 
+    # row table
+    rt = Tables.rowtable(t)
+    T = Reject(r"y")
+    n, c = apply(T, rt)
+    @test Tables.columnnames(n) == [:x1, :x2]
+    rtₒ = revert(T, n, c)
+    @test rt == rtₒ
+
     # throws: Reject without arguments
     @test_throws ArgumentError Reject()
     @test_throws ArgumentError Reject(())
@@ -303,8 +319,18 @@
     @test Tables.getcolumn(s, 1) == Tables.getcolumn(cols, 1)
     @test Tables.getcolumn(s, 3) == Tables.getcolumn(cols, :e)
 
+    # row table
+    rt = Tables.rowtable(t)
+    s = TableTransforms.TableSelection(rt, [:a, :b, :e])
+    cols = Tables.columns(rt)
+    @test Tables.getcolumn(s, :a) == Tables.getcolumn(cols, :a)
+    @test Tables.getcolumn(s, 1) == Tables.getcolumn(cols, 1)
+    @test Tables.getcolumn(s, 3) == Tables.getcolumn(cols, :e)
+
     # throws
     @test_throws AssertionError TableTransforms.TableSelection(t, [:a, :b, :z])
+    s = TableTransforms.TableSelection(t, [:a, :b, :e])
+    @test_throws ErrorException Tables.getcolumn(s, :f)
   end
 
   @testset "Filter" begin
@@ -379,6 +405,14 @@
     n1, c1 = apply(T, t)
     n2 = reapply(T, t, c1)
     @test n1 == n2
+
+    # row table
+    rt = Tables.rowtable(t)
+    T = Filter(row -> row.b == 4 || row.f == 4)
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test rt == rtₒ
   end
 
   @testset "DropMissing" begin
@@ -558,6 +592,16 @@
     n2 = reapply(T, t, c1)
     @test n1 == n2
 
+    # row table
+    rt = Tables.rowtable(t)
+    T = DropMissing()
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    for (row, rowₒ) in zip(rt, rtₒ)
+      @test isequalmissing(row, rowₒ)
+    end
+
     # throws: empty tuple
     @test_throws ArgumentError DropMissing(())
 
@@ -621,6 +665,14 @@
     tₒ = revert(T, n, c)
     @test t == tₒ
 
+    # row table
+    rt = Tables.rowtable(t)
+    T = Rename(:a => :x, :c => :y)
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test rt == rtₒ
+
     # reapply test
     T = Rename(:b => :x, :d => :y)
     n1, c1 = apply(T, t)
@@ -707,6 +759,14 @@
     tₒ = revert(T, n, c)
     @test t == tₒ
 
+    # row table
+    rt = Tables.rowtable(t)
+    T = Replace(3 => -3, 2.5 => 2.0)
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test rt == rtₒ
+
     # throws
     @test_throws ArgumentError Replace()
   end
@@ -753,7 +813,17 @@
     @test ntypes[4] == Int
     @test ntypes[5] == Int
     @test ntypes[6] == Int
-    @test ttypes == Tables.schema(tₒ).types    
+    @test ttypes == Tables.schema(tₒ).types  
+    
+    # row table
+    rt = Tables.rowtable(t)
+    T = Coalesce(0)
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    for (row, rowₒ) in zip(rt, rtₒ)
+      @test isequalmissing(row, rowₒ)
+    end
   end
   
   @testset "Coerce" begin
@@ -779,6 +849,14 @@
     tₒ = revert(T, n, c)
     @test eltype(tₒ.x1) == eltype(t.x1)
     @test eltype(tₒ.x2) == eltype(t.x2)
+
+    # row table
+    rt = Tables.rowtable(t)
+    T = Coerce(:x1 => Count, :x2 => Count)
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test rt == rtₒ
   end
 
   @testset "Identity" begin
@@ -790,6 +868,14 @@
     @test t == n
     tₒ = revert(T, n, c)
     @test t == tₒ
+
+    # row table
+    rt = Tables.rowtable(t)
+    T = Identity()
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test rt == rtₒ
   end
 
   @testset "Center" begin
@@ -813,6 +899,14 @@
 
       @test_reference joinpath(datadir, "center.png") p
     end
+
+    # row table
+    rt = Tables.rowtable(t)
+    T = Center()
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test Tables.matrix(rt) ≈ Tables.matrix(rtₒ)
   end
 
   @testset "Scale" begin
@@ -848,6 +942,14 @@
 
       @test_reference joinpath(datadir, "scale.png") p
     end
+
+    # row table
+    rt = Tables.rowtable(t)
+    T = Scale()
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test Tables.matrix(rt) ≈ Tables.matrix(rtₒ)
   end
 
   @testset "ZScore" begin
@@ -874,6 +976,14 @@
 
       @test_reference joinpath(datadir, "zscore.png") p
     end
+
+    # row table
+    rt = Tables.rowtable(t)
+    T = ZScore()
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test Tables.matrix(rt) ≈ Tables.matrix(rtₒ)
   end
 
   @testset "Quantile" begin
@@ -893,6 +1003,16 @@
     @test n.y != y
     tₒ = revert(T, n, c)
     @test tₒ.x == t.x
+
+    # row table
+    rt = Tables.rowtable(t)
+    T = Quantile()
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    for (row, rowₒ) in zip(rt, rtₒ)
+      @test row.x == rowₒ.x
+    end
   end
 
   @testset "Functional" begin
@@ -1001,6 +1121,17 @@
     T = Functional("x" => abs, "y" => sin)
     @test isrevertible(T) == false
 
+    # row table
+    x = π*rand(1500)
+    y = π*rand(1500)
+    t = Table(; x, y)
+    rt = Tables.rowtable(t)
+    T = Functional(cos)
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test Tables.matrix(rt) ≈ Tables.matrix(rtₒ)
+
     # throws
     @test_throws ArgumentError Functional()
     t = Table(x = rand(15), y = rand(15))
@@ -1081,6 +1212,17 @@
       @test_reference joinpath(datadir, "eigenanalysis-1.png") p
       @test_reference joinpath(datadir, "eigenanalysis-2.png") q
     end
+
+    # row table
+    x = rand(Normal(0, 10), 1500)
+    y = x + rand(Normal(0, 2), 1500)
+    t = Table(; x, y)
+    rt = Tables.rowtable(t)
+    T = EigenAnalysis(:V)
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test Tables.matrix(rt) ≈ Tables.matrix(rtₒ)
   end
 
   @testset "Sequential" begin
@@ -1108,6 +1250,14 @@
     n1, c1 = apply(T, t)
     n2 = reapply(T, t, c1)
     @test n1 == n2
+
+    # row table
+    rt = Tables.rowtable(t)
+    T = Scale(low=0.2, high=0.8) → EigenAnalysis(:VDV)
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test Tables.matrix(rt) ≈ Tables.matrix(rtₒ)
   end
 
   @testset "Parallel" begin
@@ -1149,6 +1299,18 @@
     n1, c1 = apply(T, t)
     n2 = reapply(T, t, c1)
     @test n1 == n2
+
+    # row table
+    x = rand(Normal(0, 10), 1500)
+    y = x + rand(Normal(0, 2), 1500)
+    z = y + rand(Normal(0, 5), 1500)
+    t = Table(; x, y, z)
+    rt = Tables.rowtable(t)
+    T = Scale(low=0.3, high=0.6) ⊔ EigenAnalysis(:VDV)
+    n, c = apply(T, rt)
+    @test Tables.isrowtable(n)
+    rtₒ = revert(T, n, c)
+    @test Tables.matrix(rt) ≈ Tables.matrix(rtₒ)
   end
 
   @testset "Miscellaneous" begin