Repository: JuliaML/TableTransforms.jl (julia)

Issue:
Add `Unitify` transform to add units to columns
It would be nice to be able to save units in column names according to a given standard, and provide transforms to "unitfy" the columns for the user. For example, we could follow the convention that `T [K]` the variable `T` has units `K`. We could then introduce a transform `Unitfy` that removes the suffix `[K]` and adds the corresponding unit with Unitful.jl to the array of values.

Pull request:
closes #231

Patch:
diff --git a/docs/src/transforms.md b/docs/src/transforms.md
index 0c1499b8..f0dab558 100644
--- a/docs/src/transforms.md
+++ b/docs/src/transforms.md
@@ -104,6 +104,12 @@ DropConstant
 AbsoluteUnits
 ```
 
+## Unitify
+
+```@docs
+Unitify
+```
+
 ## Map
 
 ```@docs
diff --git a/src/TableTransforms.jl b/src/TableTransforms.jl
index 7b557ffa..2e7ac395 100644
--- a/src/TableTransforms.jl
+++ b/src/TableTransforms.jl
@@ -65,6 +65,7 @@ export
   DropExtrema,
   DropUnits,
   AbsoluteUnits,
+  Unitify,
   Map,
   Replace,
   Coalesce,
diff --git a/src/transforms.jl b/src/transforms.jl
index 8ce9e6f1..f0aa7af4 100644
--- a/src/transforms.jl
+++ b/src/transforms.jl
@@ -279,6 +279,7 @@ include("transforms/dropextrema.jl")
 include("transforms/dropunits.jl")
 include("transforms/dropconstant.jl")
 include("transforms/absoluteunits.jl")
+include("transforms/unitify.jl")
 include("transforms/map.jl")
 include("transforms/replace.jl")
 include("transforms/coalesce.jl")
diff --git a/src/transforms/unitify.jl b/src/transforms/unitify.jl
new file mode 100644
index 00000000..c40be76e
--- /dev/null
+++ b/src/transforms/unitify.jl
@@ -0,0 +1,67 @@
+# ------------------------------------------------------------------
+# Licensed under the MIT License. See LICENSE in the project root.
+# ------------------------------------------------------------------
+
+"""
+    Unitify()
+
+Add units to columns of the table using bracket syntax.
+A column named `col [unit]` will be renamed to a unitful
+column `col` with a valid `unit` from Unitful.jl.
+
+In the case that the `unit` is not recognized by Unitful.jl, 
+no units are added. Empty brackets are also allowed to represent
+columns without units, e.g. `col []`.
+"""
+struct Unitify <: StatelessFeatureTransform end
+
+isrevertible(::Type{Unitify}) = true
+
+function _unitify(name)
+  m = match(r"(.*)\[(.*)\]", string(name))
+  if !isnothing(m)
+    namestr, unitstr = m.captures
+    newname = Symbol(strip(namestr))
+    unit = if !isempty(unitstr)
+      try
+        uparse(unitstr)
+      catch
+        @warn "the unit \"$unitstr\" is not valid"
+        NoUnits
+      end
+    else
+      NoUnits
+    end
+    newname, unit
+  else
+    name, NoUnits
+  end
+end
+
+function applyfeat(::Unitify, feat, prep)
+  cols = Tables.columns(feat)
+  names = Tables.columnnames(cols)
+
+  pairs = map(names) do name
+    x = Tables.getcolumn(cols, name)
+    newname, unit = _unitify(name)
+    newname => x * unit
+  end
+
+  newfeat = (; pairs...) |> Tables.materializer(feat)
+  newfeat, names
+end
+
+function revertfeat(::Unitify, newfeat, fcache)
+  cols = Tables.columns(newfeat)
+  names = Tables.columnnames(cols)
+
+  onames = fcache
+  ocolumns = map(names) do name
+    x = Tables.getcolumn(cols, name)
+    ustrip.(x)
+  end
+
+  𝒯 = (; zip(onames, ocolumns)...)
+  𝒯 |> Tables.materializer(newfeat)
+end


Test patch:
diff --git a/test/shows.jl b/test/shows.jl
index 346c9dbb..b20bba84 100644
--- a/test/shows.jl
+++ b/test/shows.jl
@@ -217,6 +217,32 @@
     @test iostr == "DropConstant transform"
   end
 
+  @testset "AbsoluteUnits" begin
+    T = AbsoluteUnits(:a, :b, :c)
+
+    # compact mode
+    iostr = sprint(show, T)
+    @test iostr == "AbsoluteUnits([:a, :b, :c])"
+
+    # full mode
+    iostr = sprint(show, MIME("text/plain"), T)
+    @test iostr == """
+    AbsoluteUnits transform
+    └─ selector = [:a, :b, :c]"""
+  end
+
+  @testset "Unitify" begin
+    T = Unitify()
+
+    # compact mode
+    iostr = sprint(show, T)
+    @test iostr == "Unitify()"
+
+    # full mode
+    iostr = sprint(show, MIME("text/plain"), T)
+    @test iostr == "Unitify transform"
+  end
+
   @testset "Map" begin
     fun = (a, b) -> 2a + b
     T = Map(:a => sin, [:a, :b] => fun => :c)
diff --git a/test/transforms.jl b/test/transforms.jl
index d536235f..0873c9bb 100644
--- a/test/transforms.jl
+++ b/test/transforms.jl
@@ -13,6 +13,7 @@ transformfiles = [
   "dropunits.jl",
   "dropconstant.jl",
   "absoluteunits.jl",
+  "unitify.jl",
   "map.jl",
   "replace.jl",
   "coalesce.jl",
diff --git a/test/transforms/unitify.jl b/test/transforms/unitify.jl
new file mode 100644
index 00000000..db323075
--- /dev/null
+++ b/test/transforms/unitify.jl
@@ -0,0 +1,45 @@
+@testset "Unitify" begin
+  @test isrevertible(Unitify())
+
+  anm = Symbol("a [m]")
+  bnm = :b
+  cnm = Symbol("c [km/hr]")
+  dnm = :d
+  enm = Symbol("e [°C]")
+  a = rand(10)
+  b = rand(10)
+  c = rand(10)
+  d = rand(10)
+  e = rand(10)
+  t = Table(; anm => a, bnm => b, cnm => c, dnm => d, enm => e)
+
+  T = Unitify()
+  n, c = apply(T, t)
+  @test Tables.schema(n).names == (:a, :b, :c, :d, :e)
+  @test unit(eltype(n.a)) === u"m"
+  @test unit(eltype(n.b)) === NoUnits
+  @test unit(eltype(n.c)) === u"km/hr"
+  @test unit(eltype(n.d)) === NoUnits
+  @test unit(eltype(n.e)) === u"°C"
+  tₒ = revert(T, n, c)
+  @test tₒ == t
+
+  # no units
+  t = Table(; Symbol("a []") => rand(10), Symbol("b [NoUnits]") => rand(10))
+  T = Unitify()
+  n, c = apply(T, t)
+  @test Tables.schema(n).names == (:a, :b)
+  @test unit(eltype(n.a)) === NoUnits
+  @test unit(eltype(n.b)) === NoUnits
+  tₒ = revert(T, n, c)
+  @test tₒ == t
+
+  # invalid unit name
+  t = Table(; Symbol("a [test]") => rand(10))
+  T = Unitify()
+  n, c = apply(T, t)
+  @test Tables.schema(n).names == (:a,)
+  @test unit(eltype(n.a)) === NoUnits
+  tₒ = revert(T, n, c)
+  @test tₒ == t
+end