Repository: biolab/orange3

Issue:
Feature Constructor: easier categorical features
Categorical features are hard to create, because currently:
- the user has to list all possible values
- and then the expression has to return indices for the list of values

I propose that the expression could directly return strings and that possible values were automatically created.


Discussion:
@markotoplak, you mean that a list of possible values would be created from the values returned by the function on the given input data? I like the idea, but what happens if the function returns some values that don't occur on the input data?

Another thought: it may be easier for a function to return int's than strings. (But then it can always use those ints as indices into an array.)
I was wrong. Indices are not the only way! I tried creating a feature with values A and B on iris. So my list of values was: `A, B`

Works:
- `0 if sepal_length < 5 else 1` (as shown in the documentation) 
- `A if sepal_length < 5 else B` (surprisingly! I never dared trying it)

Does not work:
- `"A" if sepal_length < 5 else "B"`

I think we should make the last example work.

Then, the list of values could be optional: it could be obtained by executing the expression on the data. For new data (in later domain transformations), if the expression returned a new value, we should make it a "?".
I quite often want to create categorical variables from text, say:

    text.split(';')[0]

But then I have to list all possible values of this attribute without often knowing what they might be. Perhaps in such cases categorical variables aren't the best option ("If you can't list them, don't have them."), but still...

Two other issues with Feature Constructor.

- It doesn't support creating datetime variables. https://github.com/biolab/orange3/issues/2900
- Sometimes one would like to define a new variable based on all existing variables say, sum of values or count non-zero). Currently not possible.





Resolving patch:
diff --git a/Orange/widgets/data/owfeatureconstructor.py b/Orange/widgets/data/owfeatureconstructor.py
--- a/Orange/widgets/data/owfeatureconstructor.py
+++ b/Orange/widgets/data/owfeatureconstructor.py
@@ -25,8 +25,7 @@
 from AnyQt.QtWidgets import (
     QSizePolicy, QAbstractItemView, QComboBox, QFormLayout, QLineEdit,
     QHBoxLayout, QVBoxLayout, QStackedWidget, QStyledItemDelegate,
-    QPushButton, QMenu, QListView, QFrame
-)
+    QPushButton, QMenu, QListView, QFrame, QLabel)
 from AnyQt.QtGui import QKeySequence
 from AnyQt.QtCore import Qt, pyqtSignal as Signal, pyqtProperty as Property
 
@@ -45,6 +44,9 @@
 ContinuousDescriptor = \
     namedtuple("ContinuousDescriptor",
                ["name", "expression", "number_of_decimals"])
+DateTimeDescriptor = \
+    namedtuple("DateTimeDescriptor",
+               ["name", "expression"])
 DiscreteDescriptor = \
     namedtuple("DiscreteDescriptor",
                ["name", "expression", "values", "ordered"])
@@ -58,6 +60,10 @@ def make_variable(descriptor, compute_value):
             descriptor.name,
             descriptor.number_of_decimals,
             compute_value)
+    if isinstance(descriptor, DateTimeDescriptor):
+        return Orange.data.TimeVariable(
+            descriptor.name,
+            compute_value=compute_value, have_date=True, have_time=True)
     elif isinstance(descriptor, DiscreteDescriptor):
         return Orange.data.DiscreteVariable(
             descriptor.name,
@@ -115,8 +121,8 @@ def __init__(self, *args, **kwargs):
                                    QSizePolicy.Fixed)
         )
         self.expressionedit = QLineEdit(
-            placeholderText="Expression..."
-        )
+            placeholderText="Expression...",
+            toolTip=self.ExpressionTooltip)
 
         self.attrs_model = itemmodels.VariableListModel(
             ["Select Feature"], parent=self)
@@ -222,6 +228,7 @@ def insert_into_expression(self, what):
 
 
 class ContinuousFeatureEditor(FeatureEditor):
+    ExpressionTooltip = "A numeric expression"
 
     def editorData(self):
         return ContinuousDescriptor(
@@ -231,15 +238,37 @@ def editorData(self):
         )
 
 
+class DateTimeFeatureEditor(FeatureEditor):
+    ExpressionTooltip = \
+        "Result must be a string in ISO-8601 format " \
+        "(e.g. 2019-07-30T15:37:27 or a part thereof),\n" \
+        "or a number of seconds since Jan 1, 1970."
+
+    def editorData(self):
+        return DateTimeDescriptor(
+            name=self.nameedit.text(),
+            expression=self.expressionedit.text()
+        )
+
+
 class DiscreteFeatureEditor(FeatureEditor):
+    ExpressionTooltip = \
+        "Result must be a string, if values are not explicitly given\n" \
+        "or a zero-based integer indices into a list of values given below."
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.valuesedit = QLineEdit()
+        tooltip = \
+            "If values are given, above expression must return zero-based " \
+            "integer indices into that list."
+        self.valuesedit = QLineEdit(placeholderText="A, B ...", toolTip=tooltip)
         self.valuesedit.textChanged.connect(self._invalidate)
 
         layout = self.layout()
-        layout.addRow(self.tr("Values"), self.valuesedit)
+        label = QLabel(self.tr("Values (optional)"))
+        label.setToolTip(tooltip)
+        layout.addRow(label, self.valuesedit)
 
     def setEditorData(self, data, domain):
         self.valuesedit.setText(
@@ -260,6 +289,8 @@ def editorData(self):
 
 
 class StringFeatureEditor(FeatureEditor):
+    ExpressionTooltip = "A string expression"
+
     def editorData(self):
         return StringDescriptor(
             name=self.nameedit.text(),
@@ -270,6 +301,7 @@ def editorData(self):
 _VarMap = {
     DiscreteDescriptor: vartype(Orange.data.DiscreteVariable()),
     ContinuousDescriptor: vartype(Orange.data.ContinuousVariable()),
+    DateTimeDescriptor: vartype(Orange.data.TimeVariable()),
     StringDescriptor: vartype(Orange.data.StringVariable())
 }
 
@@ -347,6 +379,7 @@ class Outputs:
 
     EDITORS = [
         (ContinuousDescriptor, ContinuousFeatureEditor),
+        (DateTimeDescriptor, DateTimeFeatureEditor),
         (DiscreteDescriptor, DiscreteFeatureEditor),
         (StringDescriptor, StringFeatureEditor)
     ]
@@ -412,14 +445,19 @@ def generate_newname(fmt):
         disc = menu.addAction("Categorical")
         disc.triggered.connect(
             lambda: self.addFeature(
-                DiscreteDescriptor(generate_newname("D{}"), "",
-                                   ("A", "B"), False))
+                DiscreteDescriptor(generate_newname("D{}"), "", (), False))
         )
         string = menu.addAction("Text")
         string.triggered.connect(
             lambda: self.addFeature(
                 StringDescriptor(generate_newname("S{}"), ""))
         )
+        datetime = menu.addAction("Date/Time")
+        datetime.triggered.connect(
+            lambda: self.addFeature(
+                DateTimeDescriptor(generate_newname("T{}"), ""))
+        )
+
         menu.addSeparator()
         self.duplicateaction = menu.addAction("Duplicate Selected Variable")
         self.duplicateaction.triggered.connect(self.duplicateFeature)
@@ -593,6 +631,11 @@ def validate(source):
         return final
 
     def apply(self):
+        def report_error(err):
+            log = logging.getLogger(__name__)
+            log.error("", exc_info=True)
+            self.error("".join(format_exception_only(type(err), err)).rstrip())
+
         self.Error.clear()
 
         if self.data is None:
@@ -600,8 +643,12 @@ def apply(self):
 
         desc = list(self.featuremodel)
         desc = self._validate_descriptors(desc)
-        source_vars = self.data.domain.variables + self.data.domain.metas
-        new_variables = construct_variables(desc, source_vars)
+        try:
+            new_variables = construct_variables(desc, self.data)
+        # user's expression can contain arbitrary errors
+        except Exception as err:  # pylint: disable=broad-except
+            report_error(err)
+            return
 
         attrs = [var for var in new_variables if var.is_primitive()]
         metas = [var for var in new_variables if not var.is_primitive()]
@@ -616,9 +663,7 @@ def apply(self):
         # user's expression can contain arbitrary errors
         # pylint: disable=broad-except
         except Exception as err:
-            log = logging.getLogger(__name__)
-            log.error("", exc_info=True)
-            self.error("".join(format_exception_only(type(err), err)).rstrip())
+            report_error(err)
             return
         disc_attrs_not_ok = self.check_attrs_values(
             [var for var in attrs if var.is_discrete], data)
@@ -637,6 +682,8 @@ def send_report(self):
                     "; ordered" * feature.ordered)
             elif isinstance(feature, ContinuousDescriptor):
                 items[feature.name] = "{} (numeric)".format(feature.expression)
+            elif isinstance(feature, DateTimeDescriptor):
+                items[feature.name] = "{} (date/time)".format(feature.expression)
             else:
                 items[feature.name] = "{} (text)".format(feature.expression)
         self.report_items(
@@ -815,11 +862,12 @@ def validate_exp(exp):
         raise ValueError(exp)
 
 
-def construct_variables(descriptions, source_vars):
+def construct_variables(descriptions, data):
     # subs
     variables = []
+    source_vars = data.domain.variables + data.domain.metas
     for desc in descriptions:
-        _, func = bind_variable(desc, source_vars)
+        desc, func = bind_variable(desc, source_vars, data)
         var = make_variable(desc, func)
         variables.append(var)
     return variables
@@ -832,7 +880,7 @@ def sanitized_name(name):
     return sanitized
 
 
-def bind_variable(descriptor, env):
+def bind_variable(descriptor, env, data):
     """
     (descriptor, env) ->
         (descriptor, (instance -> value) | (table -> value list))
@@ -847,10 +895,35 @@ def bind_variable(descriptor, env):
                    if name in variables]
 
     values = {}
+    cast = None
+    nan = float("nan")
+
     if isinstance(descriptor, DiscreteDescriptor):
-        values = [sanitized_name(v) for v in descriptor.values]
-        values = {name: i for i, name in enumerate(values)}
-    return descriptor, FeatureFunc(descriptor.expression, source_vars, values)
+        if not descriptor.values:
+            str_func = FeatureFunc(descriptor.expression, source_vars)
+            values = sorted({str(x) for x in str_func(data)})
+            values = {name: i for i, name in enumerate(values)}
+            descriptor = descriptor._replace(values=values)
+
+            def cast(x):  # pylint: disable=function-redefined
+                return values.get(x, nan)
+
+        else:
+            values = [sanitized_name(v) for v in descriptor.values]
+            values = {name: i for i, name in enumerate(values)}
+
+    if isinstance(descriptor, DateTimeDescriptor):
+        parse = Orange.data.TimeVariable("_").parse
+
+        def cast(e):  # pylint: disable=function-redefined
+            if isinstance(e, (int, float)):
+                return e
+            if e == "" or e is None:
+                return np.nan
+            return parse(e)
+
+    func = FeatureFunc(descriptor.expression, source_vars, values, cast)
+    return descriptor, func
 
 
 def make_lambda(expression, args, env=None):
@@ -978,23 +1051,33 @@ class FeatureFunc:
     extra_env : Optional[Dict[str, Any]]
         Extra environment specifying constant values to be made available
         in expression. It must not shadow names in `args`
+    cast: Optional[Callable]
+        A function for casting the expressions result to the appropriate
+        type (e.g. string representation of date/time variables to floats)
     """
-    def __init__(self, expression, args, extra_env=None):
+    def __init__(self, expression, args, extra_env=None, cast=None):
         self.expression = expression
         self.args = args
         self.extra_env = dict(extra_env or {})
         self.func = make_lambda(ast.parse(expression, mode="eval"),
                                 [name for name, _ in args], self.extra_env)
+        self.cast = cast
 
     def __call__(self, instance, *_):
         if isinstance(instance, Orange.data.Table):
             return [self(inst) for inst in instance]
         else:
-            args = [instance[var] for _, var in self.args]
-            return self.func(*args)
+            args = [str(instance[var])
+                    if instance.domain[var].is_string else instance[var]
+                    for _, var in self.args]
+            y = self.func(*args)
+            if self.cast:
+                y = self.cast(y)
+            return y
 
     def __reduce__(self):
-        return type(self), (self.expression, self.args, self.extra_env)
+        return type(self), (self.expression, self.args,
+                            self.extra_env, self.cast)
 
     def __repr__(self):
         return "{0.__name__}{1!r}".format(*self.__reduce__())


Test patch:
diff --git a/Orange/widgets/data/tests/test_owfeatureconstructor.py b/Orange/widgets/data/tests/test_owfeatureconstructor.py
--- a/Orange/widgets/data/tests/test_owfeatureconstructor.py
+++ b/Orange/widgets/data/tests/test_owfeatureconstructor.py
@@ -8,14 +8,15 @@
 import numpy as np
 
 from Orange.data import (Table, Domain, StringVariable,
-                         ContinuousVariable, DiscreteVariable)
+                         ContinuousVariable, DiscreteVariable, TimeVariable)
 from Orange.widgets.tests.base import WidgetTest
 from Orange.widgets.utils import vartype
 from Orange.widgets.utils.itemmodels import PyListModel
 from Orange.widgets.data.owfeatureconstructor import (
     DiscreteDescriptor, ContinuousDescriptor, StringDescriptor,
     construct_variables, OWFeatureConstructor,
-    FeatureEditor, DiscreteFeatureEditor, FeatureConstructorHandler)
+    FeatureEditor, DiscreteFeatureEditor, FeatureConstructorHandler,
+    DateTimeDescriptor)
 
 from Orange.widgets.data.owfeatureconstructor import (
     freevars, validate_exp, FeatureFunc
@@ -34,7 +35,7 @@ def test_construct_variables_discrete(self):
                                 values=values, ordered=False)]
         )
         data = Table(Domain(list(data.domain.attributes) +
-                            construct_variables(desc, data.domain.variables),
+                            construct_variables(desc, data),
                             data.domain.class_vars,
                             data.domain.metas), data)
         self.assertTrue(isinstance(data.domain[name], DiscreteVariable))
@@ -42,6 +43,26 @@ def test_construct_variables_discrete(self):
         for i in range(3):
             self.assertEqual(data[i * 50, name], values[i])
 
+    def test_construct_variables_discrete_no_values(self):
+        data = Table("iris")
+        name = 'Discrete Variable'
+        expression = "str(iris)[-1]"  # last letter - a or r
+        values = ()
+        desc = PyListModel(
+            [DiscreteDescriptor(name=name, expression=expression,
+                                values=values, ordered=False)]
+        )
+        data = Table(Domain(list(data.domain.attributes) +
+                            construct_variables(desc, data),
+                            data.domain.class_vars,
+                            data.domain.metas), data)
+        newvar = data.domain[name]
+        self.assertTrue(isinstance(newvar, DiscreteVariable))
+        self.assertEqual(set(data.domain[name].values), set("ar"))
+        for i in range(3):
+            inst = data[i * 50]
+            self.assertEqual(str(inst[name]), str(inst["iris"])[-1])
+
     def test_construct_variables_continuous(self):
         data = Table("iris")
         name = 'Continuous Variable'
@@ -51,7 +72,7 @@ def test_construct_variables_continuous(self):
                                   number_of_decimals=2)]
         )
         data = Table(Domain(list(data.domain.attributes) +
-                            construct_variables(featuremodel, data.domain.variables),
+                            construct_variables(featuremodel, data),
                             data.domain.class_vars,
                             data.domain.metas), data)
         self.assertTrue(isinstance(data.domain[name], ContinuousVariable))
@@ -59,6 +80,22 @@ def test_construct_variables_continuous(self):
             self.assertEqual(data[i * 50, name],
                              pow(data[i * 50, 0] + data[i * 50, 1], 2))
 
+    def test_construct_variables_datetime(self):
+        data = Table("housing")
+        name = 'Date'
+        expression = '"2019-07-{:02}".format(int(MEDV/3))'
+        featuremodel = PyListModel(
+            [DateTimeDescriptor(name=name, expression=expression)]
+        )
+        data = Table(Domain(list(data.domain.attributes) +
+                            construct_variables(featuremodel, data),
+                            data.domain.class_vars,
+                            data.domain.metas), data)
+        self.assertTrue(isinstance(data.domain[name], TimeVariable))
+        for row in data:
+            self.assertEqual("2019-07-{:02}".format(int(row["MEDV"] / 3)),
+                             str(row["Date"])[:10])
+
     def test_construct_variables_string(self):
         data = Table("iris")
         name = 'String Variable'
@@ -69,7 +106,7 @@ def test_construct_variables_string(self):
         data = Table(Domain(data.domain.attributes,
                             data.domain.class_vars,
                             list(data.domain.metas) +
-                            construct_variables(desc, data.domain.variables)),
+                            construct_variables(desc, data)),
                      data)
         self.assertTrue(isinstance(data.domain[name], StringVariable))
         for i in range(3):
@@ -86,7 +123,7 @@ def test_construct_numeric_names():
                                   expression="_0_1 + _1",
                                   number_of_decimals=3)]
         )
-        nv = construct_variables(desc, data.domain.variables)
+        nv = construct_variables(desc, data)
         ndata = Table(Domain(nv, None), data)
         np.testing.assert_array_equal(ndata.X[:, 0],
                                       data.X[:, :2].sum(axis=1))
@@ -204,26 +241,40 @@ def validate_(source):
 
 class FeatureFuncTest(unittest.TestCase):
     def test_reconstruct(self):
-        f = FeatureFunc("a * x + c", [("x", "x")], {"a": 2, "c": 10})
-        self.assertEqual(f({"x": 2}), 14)
+        iris = Table("iris")
+        inst1 = iris[0]
+        val1 = 2 * inst1["sepal width"] + 10
+        inst2 = iris[100]
+        val2 = 2 * inst2["sepal width"] + 10
+
+        f = FeatureFunc("a * sepal_width + c",
+                        [("sepal_width", iris.domain["sepal width"])],
+                        {"a": 2, "c": 10})
+        self.assertAlmostEqual(f(inst1), val1)
         f1 = pickle.loads(pickle.dumps(f))
-        self.assertEqual(f1({"x": 2}), 14)
+        self.assertAlmostEqual(f1(inst1), val1)
         fc = copy.copy(f)
-        self.assertEqual(fc({"x": 3}), 16)
+        self.assertEqual(fc(inst2), val2)
 
     def test_repr(self):
         self.assertEqual(repr(FeatureFunc("a + 1", [("a", 2)])),
-                         "FeatureFunc('a + 1', [('a', 2)], {})")
+                         "FeatureFunc('a + 1', [('a', 2)], {}, None)")
 
     def test_call(self):
-        f = FeatureFunc("a + 1", [("a", "a")])
-        self.assertEqual(f({"a": 2}), 3)
-
         iris = Table("iris")
         f = FeatureFunc("sepal_width + 10",
                         [("sepal_width", iris.domain["sepal width"])])
         r = f(iris)
         np.testing.assert_array_equal(r, iris.X[:, 1] + 10)
+        self.assertEqual(f(iris[0]), iris[0]["sepal width"] + 10)
+
+    def test_string_casting(self):
+        zoo = Table("zoo")
+        f = FeatureFunc("name[0]",
+                        [("name", zoo.domain["name"])])
+        r = f(zoo)
+        self.assertEqual(r, [x[0] for x in zoo.metas[:, 0]])
+        self.assertEqual(f(zoo[0]), str(zoo[0, "name"])[0])
 
 
 class OWFeatureConstructorTests(WidgetTest):
@@ -257,7 +308,7 @@ def test_discrete_no_values(self):
         self.widget.setData(data)
         discreteFeatureEditor = DiscreteFeatureEditor()
 
-        discreteFeatureEditor.valuesedit.setText("")
+        discreteFeatureEditor.valuesedit.setText("A")
         discreteFeatureEditor.nameedit.setText("D1")
         discreteFeatureEditor.expressionedit.setText("iris")
         self.widget.addFeature(