Repository: JuliaML/TableTransforms.jl
Pull request #41 (2022-04-12 00:28:30)

Rename `_filter` to `choose` and create colspec.jl file in test folder
This PR renames `_filter` to `choose` and moves ColSpec tests to colspec.jl file.


Patch:
diff --git a/README.md b/README.md
index b58b203..d733017 100644
--- a/README.md
+++ b/README.md
@@ -150,7 +150,6 @@ Please check the docstrings for additional information.
 | `Filter` | Row filtering |
 | `DropMissing` | Drop missings |
 | `Rename` | Column renaming |
-| `Coalesce` | Replace missings |
 | `Identity` | Identity transform |
 | `Center` | Mean removal |
 | `Scale` | Interval scaling |
diff --git a/src/TableTransforms.jl b/src/TableTransforms.jl
index 5ffa6a0..c2f02c5 100644
--- a/src/TableTransforms.jl
+++ b/src/TableTransforms.jl
@@ -36,7 +36,6 @@ export
   Filter,
   DropMissing,
   Rename,
-  Coalesce,
   Identity,
   Center,
   Scale,
diff --git a/src/colspec.jl b/src/colspec.jl
index de9504c..30ef15c 100644
--- a/src/colspec.jl
+++ b/src/colspec.jl
@@ -10,7 +10,7 @@ const ColSelector = Union{Symbol,Integer,AbstractString}
 
 `ColSpec` is a union of types used to filter columns. 
 The `ColSpec` type together with the `ColSelector` union type and 
-the `_filter` internal function form the ColSpec interface.
+the `choose` function form the ColSpec interface.
 
 To implement the ColSpec interface, the following steps must be performed:
 
@@ -21,12 +21,12 @@ struct MyTransform{S<:ColSpec,#= other type params =#}
   # other fileds
 end
 ```
-2. use `_filter(colspec, names)` internal function in apply:
+2. use `choose(colspec, names)` function in apply:
 ```julia
 function apply(transform::MyTransform, table)
   names = Tables.columnnames(table)
   # selected column names
-  snames = _filter(transform.colspec, names)
+  snames = choose(transform.colspec, names)
   # code...
 end
 ```
@@ -41,33 +41,33 @@ end
 """
 const ColSpec = Union{Vector{T},NTuple{N,T},Regex,Colon} where {N,T<:ColSelector}
 
-# filter table columns using colspec
-function _filter(colspec::Vector{Symbol}, names)
+# choose column names using colspec
+function choose(colspec::Vector{Symbol}, names)
   # validate columns
   @assert !isempty(colspec) "Invalid column selection."
   @assert colspec ⊆ names "Invalid column selection."
   return colspec
 end
 
-_filter(colspec::Vector{<:AbstractString}, names) = 
-  _filter(Symbol.(colspec), names)
+choose(colspec::Vector{<:AbstractString}, names) = 
+  choose(Symbol.(colspec), names)
 
-_filter(colspec::Vector{<:Integer}, names::Vector) = 
-  _filter(names[colspec], names)
+choose(colspec::Vector{<:Integer}, names::Vector) = 
+  choose(names[colspec], names)
 
-_filter(colspec::Vector{<:Integer}, names::Tuple) = 
-  _filter(colspec, collect(names))
+choose(colspec::Vector{<:Integer}, names::Tuple) = 
+  choose(colspec, collect(names))
 
-_filter(colspec::NTuple{N,<:ColSelector}, names) where {N} =
-  _filter(collect(colspec), names)
+choose(colspec::NTuple{N,<:ColSelector}, names) where {N} =
+  choose(collect(colspec), names)
 
-function _filter(colspec::Regex, names::Vector)
-  fnames = filter(col -> occursin(colspec, String(col)), names)
-  _filter(fnames, names)
+function choose(colspec::Regex, names::Vector)
+  fnames = filter(n -> occursin(colspec, String(n)), names)
+  choose(fnames, names)
 end
 
-_filter(colspec::Regex, names::Tuple) = 
-  _filter(colspec, collect(names))
+choose(colspec::Regex, names::Tuple) = 
+  choose(colspec, collect(names))
 
-_filter(::Colon, names::Vector) = names
-_filter(::Colon, names::Tuple) = collect(names)
+choose(::Colon, names::Vector) = names
+choose(::Colon, names::Tuple) = collect(names)
diff --git a/src/transforms.jl b/src/transforms.jl
index 36586ed..5f8ff97 100644
--- a/src/transforms.jl
+++ b/src/transforms.jl
@@ -214,7 +214,6 @@ end
 include("transforms/select.jl")
 include("transforms/filter.jl")
 include("transforms/rename.jl")
-include("transforms/coalesce.jl")
 include("transforms/identity.jl")
 include("transforms/center.jl")
 include("transforms/scale.jl")
diff --git a/src/transforms/coalesce.jl b/src/transforms/coalesce.jl
deleted file mode 100644
index 54ee28a..0000000
--- a/src/transforms/coalesce.jl
+++ /dev/null
@@ -1,22 +0,0 @@
-# ------------------------------------------------------------------
-# Licensed under the MIT License. See LICENSE in the project root.
-# ------------------------------------------------------------------
-
-"""
-    Coalesce(value)
-
-Replaces all missing values from the table with `value`.
-"""
-struct Coalesce{T} <: Colwise
-  value::T
-end
-
-isrevertible(::Type{<:Coalesce}) = true
-
-colcache(::Coalesce, x) = findall(ismissing, x)
-
-colapply(tramsform::Coalesce, x, c) =
-  coalesce.(x, tramsform.value)
-
-colrevert(::Coalesce, x, c) =
-  map(i -> i ∈ c ? missing : x[i], 1:length(x))
diff --git a/src/transforms/filter.jl b/src/transforms/filter.jl
index daef090..3c380eb 100644
--- a/src/transforms/filter.jl
+++ b/src/transforms/filter.jl
@@ -79,7 +79,7 @@ _nonmissing(x) = _nonmissing(eltype(x), x)
 function apply(transform::DropMissing, table)
   names = Tables.columnnames(table)
   types = Tables.schema(table).types
-  snames = _filter(transform.colspec, names)
+  snames = choose(transform.colspec, names)
   ftrans = _ftrans(transform, snames)
   newtable, fcache = apply(ftrans, table)
 
diff --git a/src/transforms/select.jl b/src/transforms/select.jl
index db389e5..5897a3b 100644
--- a/src/transforms/select.jl
+++ b/src/transforms/select.jl
@@ -69,7 +69,7 @@ isrevertible(::Type{<:Select}) = true
 function apply(transform::Select, table)
   # retrieve relevant column names
   allcols = collect(Tables.columnnames(table))
-  select  = _filter(transform.colspec, allcols)
+  select  = choose(transform.colspec, allcols)
   reject  = setdiff(allcols, select)
 
   # keep track of indices to revert later
@@ -141,7 +141,7 @@ isrevertible(::Type{<:Reject}) = true
 
 function apply(transform::Reject, table)
   allcols = Tables.columnnames(table)
-  reject  = _filter(transform.colspec, allcols)
+  reject  = choose(transform.colspec, allcols)
   select  = setdiff(allcols, reject)
   strans  = Select(select)
   newtable, scache = apply(strans, table)


Test patch:
diff --git a/test/colspec.jl b/test/colspec.jl
new file mode 100644
index 0000000..1039ae1
--- /dev/null
+++ b/test/colspec.jl
@@ -0,0 +1,85 @@
+@testset "ColSpec" begin
+  vecnames = [:a, :b, :c, :d, :e, :f]
+  tupnames = (:a, :b, :c, :d, :e, :f)
+
+  # vector of symbols
+  colspec = [:a, :c, :e]
+  snames = TableTransforms.choose(colspec, vecnames)
+  @test snames == [:a, :c, :e]
+  snames = TableTransforms.choose(colspec, tupnames)
+  @test snames == [:a, :c, :e]
+
+  # tuple of symbols
+  colspec = (:a, :c, :e)
+  snames = TableTransforms.choose(colspec, vecnames)
+  @test snames == [:a, :c, :e]
+  snames = TableTransforms.choose(colspec, tupnames)
+  @test snames == [:a, :c, :e]
+
+  # vector of strings
+  colspec = ["a", "c", "e"]
+  snames = TableTransforms.choose(colspec, vecnames)
+  @test snames == [:a, :c, :e]
+  snames = TableTransforms.choose(colspec, tupnames)
+  @test snames == [:a, :c, :e]
+
+  # tuple of strings
+  colspec = ("a", "c", "e")
+  snames = TableTransforms.choose(colspec, vecnames)
+  @test snames == [:a, :c, :e]
+  snames = TableTransforms.choose(colspec, tupnames)
+  @test snames == [:a, :c, :e]
+
+  # vector of integers
+  colspec = [1, 3, 5]
+  snames = TableTransforms.choose(colspec, vecnames)
+  @test snames == [:a, :c, :e]
+  snames = TableTransforms.choose(colspec, tupnames)
+  @test snames == [:a, :c, :e]
+
+  # tuple of integers
+  colspec = (1, 3, 5)
+  snames = TableTransforms.choose(colspec, vecnames)
+  @test snames == [:a, :c, :e]
+  snames = TableTransforms.choose(colspec, tupnames)
+  @test snames == [:a, :c, :e]
+
+  # regex
+  colspec = r"[ace]"
+  snames = TableTransforms.choose(colspec, vecnames)
+  @test snames == [:a, :c, :e]
+  snames = TableTransforms.choose(colspec, tupnames)
+  @test snames == [:a, :c, :e]
+
+  # colon
+  snames = TableTransforms.choose(:, vecnames)
+  @test snames == [:a, :b, :c, :d, :e, :f]
+  snames = TableTransforms.choose(:, tupnames)
+  @test snames == [:a, :b, :c, :d, :e, :f]
+
+  # throws
+  @test_throws AssertionError TableTransforms.choose(r"x", vecnames)
+  @test_throws AssertionError TableTransforms.choose(r"x", tupnames)
+  @test_throws AssertionError TableTransforms.choose(String[], vecnames)
+  @test_throws AssertionError TableTransforms.choose(String[], tupnames)
+  @test_throws AssertionError TableTransforms.choose(Symbol[], vecnames)
+  @test_throws AssertionError TableTransforms.choose(Symbol[], tupnames)
+
+  # type stability
+  @inferred TableTransforms.choose([:a, :b], vecnames)
+  @inferred TableTransforms.choose([:a, :b], tupnames)
+  @inferred TableTransforms.choose((:a, :b), vecnames)
+  @inferred TableTransforms.choose((:a, :b), tupnames)
+  @inferred TableTransforms.choose(["a", "b"], vecnames)
+  @inferred TableTransforms.choose(["a", "b"], tupnames)
+  @inferred TableTransforms.choose(("a", "b"), vecnames)
+  @inferred TableTransforms.choose(("a", "b"), tupnames)
+  @inferred TableTransforms.choose([1, 2], vecnames)
+  @inferred TableTransforms.choose([1, 2], tupnames)
+  @inferred TableTransforms.choose((1, 2), vecnames)
+  @inferred TableTransforms.choose((1, 2), tupnames)
+  @inferred TableTransforms.choose(r"[ab]", vecnames)
+  @inferred TableTransforms.choose(r"[ab]", tupnames)
+  @inferred TableTransforms.choose(:, vecnames)
+  @inferred TableTransforms.choose(:, tupnames)
+end
diff --git a/test/runtests.jl b/test/runtests.jl
index 48291b1..b744fe6 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -27,18 +27,18 @@ end
 Polynomial(args::T...) where {T<:Real} = Polynomial(collect(args))
 (p::Polynomial)(x) = sum(a * x^(i-1) for (i, a) in enumerate(p.coeffs))
 
-function isequalmissing(v1, v2)
-  length(v1) == length(v2) || return false
-  for (x, y) in zip(v1, v2)
+function isequalmissing(a, b)
+  length(a) == length(b) || return false
+  for (x, y) in zip(a, b)
     x === y || return false
   end
   return true
 end
 
-
 # list of tests
 testfiles = [
   "distributions.jl",
+  "colspec.jl",
   "transforms.jl"
 ]
 
diff --git a/test/transforms.jl b/test/transforms.jl
index 3497fee..6b51539 100644
--- a/test/transforms.jl
+++ b/test/transforms.jl
@@ -1,92 +1,6 @@
 @testset "Transforms" begin
   # using MersenneTwister for compatibility between Julia versions
   rng = MersenneTwister(42)
-  @testset "ColSpec" begin
-    veccols = [:a, :b, :c, :d, :e, :f]
-    tupcols = (:a, :b, :c, :d, :e, :f)
-    
-    # vector of symbols
-    colspec = [:a, :c, :e]
-    cols = TableTransforms._filter(colspec, veccols)
-    @test cols == [:a, :c, :e]
-    cols = TableTransforms._filter(colspec, tupcols)
-    @test cols == [:a, :c, :e]
-
-    # tuple of symbols
-    colspec = (:a, :c, :e)
-    cols = TableTransforms._filter(colspec, veccols)
-    @test cols == [:a, :c, :e]
-    cols = TableTransforms._filter(colspec, tupcols)
-    @test cols == [:a, :c, :e]
-
-    # vector of strings
-    colspec = ["a", "c", "e"]
-    cols = TableTransforms._filter(colspec, veccols)
-    @test cols == [:a, :c, :e]
-    cols = TableTransforms._filter(colspec, tupcols)
-    @test cols == [:a, :c, :e]
-
-    # tuple of strings
-    colspec = ("a", "c", "e")
-    cols = TableTransforms._filter(colspec, veccols)
-    @test cols == [:a, :c, :e]
-    cols = TableTransforms._filter(colspec, tupcols)
-    @test cols == [:a, :c, :e]
-
-    # vector of integers
-    colspec = [1, 3, 5]
-    cols = TableTransforms._filter(colspec, veccols)
-    @test cols == [:a, :c, :e]
-    cols = TableTransforms._filter(colspec, tupcols)
-    @test cols == [:a, :c, :e]
-
-    # tuple of integers
-    colspec = (1, 3, 5)
-    cols = TableTransforms._filter(colspec, veccols)
-    @test cols == [:a, :c, :e]
-    cols = TableTransforms._filter(colspec, tupcols)
-    @test cols == [:a, :c, :e]
-
-    # regex
-    colspec = r"[ace]"
-    cols = TableTransforms._filter(colspec, veccols)
-    @test cols == [:a, :c, :e]
-    cols = TableTransforms._filter(colspec, tupcols)
-    @test cols == [:a, :c, :e]
-
-    # colon
-    cols = TableTransforms._filter(:, veccols)
-    @test cols == [:a, :b, :c, :d, :e, :f]
-    cols = TableTransforms._filter(:, tupcols)
-    @test cols == [:a, :b, :c, :d, :e, :f]
-
-    # throws
-    @test_throws AssertionError TableTransforms._filter(r"x", veccols)
-    @test_throws AssertionError TableTransforms._filter(r"x", tupcols)
-    @test_throws AssertionError TableTransforms._filter(String[], veccols)
-    @test_throws AssertionError TableTransforms._filter(String[], tupcols)
-    @test_throws AssertionError TableTransforms._filter(Symbol[], veccols)
-    @test_throws AssertionError TableTransforms._filter(Symbol[], tupcols)
-
-    # type stability
-    @inferred TableTransforms._filter([:a, :b], veccols)
-    @inferred TableTransforms._filter([:a, :b], tupcols)
-    @inferred TableTransforms._filter((:a, :b), veccols)
-    @inferred TableTransforms._filter((:a, :b), tupcols)
-    @inferred TableTransforms._filter(["a", "b"], veccols)
-    @inferred TableTransforms._filter(["a", "b"], tupcols)
-    @inferred TableTransforms._filter(("a", "b"), veccols)
-    @inferred TableTransforms._filter(("a", "b"), tupcols)
-    @inferred TableTransforms._filter([1, 2], veccols)
-    @inferred TableTransforms._filter([1, 2], tupcols)
-    @inferred TableTransforms._filter((1, 2), veccols)
-    @inferred TableTransforms._filter((1, 2), tupcols)
-    @inferred TableTransforms._filter(r"[ab]", veccols)
-    @inferred TableTransforms._filter(r"[ab]", tupcols)
-    @inferred TableTransforms._filter(:, veccols)
-    @inferred TableTransforms._filter(:, tupcols)
-  end
-
   @testset "Select" begin
     a = rand(4000)
     b = rand(4000)
@@ -488,12 +402,10 @@
     # revert test
     @test isrevertible(T) == true
     tₒ = revert(T, n, c)
-    cols = Tables.columns(t)
-    colsₒ = Tables.columns(tₒ)
     colnames = Tables.columnnames(t)
-    for n in colnames
-      col = Tables.getcolumn(cols, n)
-      colₒ = Tables.getcolumn(colsₒ, n)
+    for colname in colnames
+      col = Tables.getcolumn(t, colname)
+      colₒ = Tables.getcolumn(tₒ, colname)
       @test isequalmissing(col, colₒ)
     end
 
@@ -714,51 +626,6 @@
     @test n1 == n2
   end
 
-  @testset "Coalesce" begin
-    a = [3, 2, missing, 4, 5, 3]
-    b = [missing, 4, 4, 5, 8, 5]
-    c = [1, 1, 6, 2, 4, missing]
-    d = [4, 3, 7, 5, 4, missing]
-    e = [missing, 5, 2, 6, 5, 2]
-    f = [4, missing, 3, 4, 5, 2]
-    t = Table(; a, b, c, d, e, f)
-
-    T = Coalesce(0)
-    n, c = apply(T, t)
-    @test n.a == [3, 2, 0, 4, 5, 3]
-    @test n.b == [0, 4, 4, 5, 8, 5]
-    @test n.c == [1, 1, 6, 2, 4, 0]
-    @test n.d == [4, 3, 7, 5, 4, 0]
-    @test n.e == [0, 5, 2, 6, 5, 2]
-    @test n.f == [4, 0, 3, 4, 5, 2]
-
-    # revert test
-    @test isrevertible(T) == true
-    tₒ = revert(T, n, c)
-    cols = Tables.columns(t)
-    colsₒ = Tables.columns(tₒ)
-    colnames = Tables.columnnames(t)
-    for n in colnames
-      col = Tables.getcolumn(cols, n)
-      colₒ = Tables.getcolumn(colsₒ, n)
-      @test isequalmissing(col, colₒ)
-    end
-
-    # table schema after apply and revert
-    T = Coalesce(0)
-    n, c = apply(T, t)
-    tₒ = revert(T, n, c)
-    ttypes = Tables.schema(t).types
-    ntypes = Tables.schema(n).types
-    @test ntypes[1] == Int
-    @test ntypes[2] == Int
-    @test ntypes[3] == Int
-    @test ntypes[4] == Int
-    @test ntypes[5] == Int
-    @test ntypes[6] == Int
-    @test ttypes == Tables.schema(tₒ).types    
-  end
-
   @testset "Identity" begin
     x = rand(4000)
     y = rand(4000)