Repository: biolab/orange3

Issue:
Concatenate from position
<!-- 
Thanks for taking the time to submit a feature request!
For the best chance at our team considering your request, please answer the following questions to the best of your ability.
-->

**What's your use case?**
For my budget data that is stored in several files (one by year), the column can vary in name (such as oct2018/10-2018/2018 October....) even if the fifht column is always the one corresponding to january, the sixth to february, etc.. I would like to associate all these files with the concatenate tool.

**What's your proposed solution?**
I would like an option on the concatenate tool to authorize the concatenation according to the field position instead of the field name.


**Are there any alternative solutions?**
Rename field before... but that's a manually task.







Discussion:
Note to self: add a third radio button, rephrase options.

Resolving patch:
diff --git a/Orange/data/table.py b/Orange/data/table.py
--- a/Orange/data/table.py
+++ b/Orange/data/table.py
@@ -1325,7 +1325,7 @@ def __repr__(self):
         return s
 
     @classmethod
-    def concatenate(cls, tables, axis=0):
+    def concatenate(cls, tables, axis=0, *, ignore_domains=None):
         """
         Concatenate tables into a new table, either vertically or horizontally.
 
@@ -1346,14 +1346,15 @@ def concatenate(cls, tables, axis=0):
         """
         if axis not in (0, 1):
             raise ValueError("invalid axis")
+        if ignore_domains is not None and axis != 0:
+            raise ValueError("'ignore_domains' is incompatible with 'axis=1'")
         if not tables:
             raise ValueError('need at least one table to concatenate')
 
         if len(tables) == 1:
             return tables[0].copy()
-
         if axis == 0:
-            conc = cls._concatenate_vertical(tables)
+            conc = cls._concatenate_vertical(tables, bool(ignore_domains))
         else:
             conc = cls._concatenate_horizontal(tables)
 
@@ -1368,7 +1369,7 @@ def concatenate(cls, tables, axis=0):
         return conc
 
     @classmethod
-    def _concatenate_vertical(cls, tables):
+    def _concatenate_vertical(cls, tables, ignore_domains=False):
         def vstack(arrs):
             return [np, sp][any(sp.issparse(arr) for arr in arrs)].vstack(arrs)
 
@@ -1387,7 +1388,8 @@ def collect(attr):
             return [getattr(arr, attr) for arr in tables]
 
         domain = tables[0].domain
-        if any(table.domain != domain for table in tables):
+        if not ignore_domains \
+                and any(table.domain != domain for table in tables):
             raise ValueError('concatenated tables must have the same domain')
 
         conc = cls.from_numpy(
diff --git a/Orange/widgets/data/owconcatenate.py b/Orange/widgets/data/owconcatenate.py
--- a/Orange/widgets/data/owconcatenate.py
+++ b/Orange/widgets/data/owconcatenate.py
@@ -7,7 +7,7 @@
 """
 from collections import OrderedDict, namedtuple, defaultdict, Counter
 from functools import reduce
-from itertools import chain, count
+from itertools import chain, count, zip_longest
 from typing import List, Optional, Sequence
 
 import numpy as np
@@ -44,6 +44,8 @@ class Outputs:
 
     class Error(widget.OWWidget.Error):
         bow_concatenation = Msg("Inputs must be of the same type.")
+        incompatible_domains = \
+            Msg("Ignoring column names requires matching column types")
 
     class Warning(widget.OWWidget.Warning):
         renamed_variables = Msg(
@@ -56,6 +58,7 @@ class Warning(widget.OWWidget.Warning):
     append_source_column: bool
     source_column_role: int
     source_attr_name: str
+    ignore_names: bool
 
     #: Domain merging operations
     MergeUnion, MergeIntersection = 0, 1
@@ -71,6 +74,8 @@ class Warning(widget.OWWidget.Warning):
     source_column_role = settings.Setting(0)
     #: User specified name for the "Source ID" attr
     source_attr_name = settings.Setting("Source ID")
+    #: Use names from the primary table, ignore others
+    ignore_names = settings.Setting(False)
 
     ignore_compute_value = settings.Setting(False)
 
@@ -90,27 +95,29 @@ def __init__(self):
         self.primary_data = None
         self._more_data_input: List[Optional[Orange.data.Table]] = []
 
-        self.mergebox = gui.vBox(self.controlArea, "Variable Merging")
-        box = gui.radioButtons(
-            self.mergebox, self, "merge_type",
-            callback=self._merge_type_changed)
-
+        self.mergebox = gui.vBox(self.controlArea, "Variable Sets Merging")
         gui.widgetLabel(
-            box, self.tr("When there is no primary table, " +
-                         "the output should contain:"))
+            self.mergebox, self.tr("When there is no primary table, " +
+                                   "the output should contain"))
 
-        for opts in self.domain_opts:
-            gui.appendRadioButton(box, self.tr(opts))
-
-        gui.separator(box)
+        gui.radioButtons(
+            gui.indentedBox(self.mergebox, 10), self, "merge_type", self.domain_opts,
+            callback=self._merge_type_changed)
 
         label = gui.widgetLabel(
-            box,
+            self.mergebox,
             self.tr("The resulting table will have a class only if there " +
                     "is no conflict between input classes."))
         label.setWordWrap(True)
 
-        gui.separator(box)
+        box = gui.vBox(self.controlArea, "Variable matching")
+        gui.checkBox(
+            box, self, "ignore_names",
+            "Use column names from the primary table,\n"
+            "and ignore names in other tables.",
+            callback=self.ignore_names_changed, stateWhenDisabled=False)
+
+        gui.separator(self.controlArea)
         gui.checkBox(
             box, self, "ignore_compute_value",
             "Treat variables with the same name as the same variable,\n"
@@ -179,12 +186,19 @@ def more_data(self) -> Sequence[Orange.data.Table]:
 
     def handleNewSignals(self):
         self.mergebox.setDisabled(self.primary_data is not None)
+        self.controls.ignore_names.setEnabled(self.primary_data is not None)
+        self.controls.ignore_compute_value.setDisabled(
+            self.primary_data is not None and self.ignore_names)
         if self.incompatible_types():
             self.Error.bow_concatenation()
         else:
             self.Error.bow_concatenation.clear()
             self.commit.now()
 
+    def ignore_names_changed(self):
+        self.controls.ignore_compute_value.setDisabled(self.ignore_names)
+        self.commit.deferred()
+
     def incompatible_types(self):
         types_ = set()
         if self.primary_data is not None:
@@ -200,6 +214,7 @@ def incompatible_types(self):
     def commit(self):
         self.Warning.renamed_variables.clear()
         self.Warning.unmergeable_attributes.clear()
+        self.Error.incompatible_domains.clear()
         tables, domain, source_var = [], None, None
         if self.primary_data is not None:
             tables = [self.primary_data] + list(self.more_data)
@@ -222,22 +237,46 @@ def commit(self):
                 get_unique_names(domain, self.source_attr_name),
                 values=names
             )
-            places = ["class_vars", "attributes", "metas"]
-            domain = add_columns(
-                domain,
-                **{places[self.source_column_role]: (source_var,)})
+            source_ids = np.array(list(flatten(
+                [i] * len(table) for i, table in enumerate(tables)))).reshape((-1, 1))
 
-        tables = [table.transform(domain) for table in tables]
-        if tables:
+        if not tables:
+            data = None
+        elif self.primary_data is not None and self.ignore_names:
+            if any(type(pv) is not type(mv) or
+                   pv.is_discrete and pv.values != mv.values
+                   for table in self.more_data
+                   for pv, mv in chain(zip_longest(domain.attributes, table.domain.attributes),
+                                       zip_longest(domain.class_vars, table.domain.class_vars),
+                                       zip_longest(domain.metas, table.domain.metas)
+                                       )
+                   ):
+                self.Error.incompatible_domains()
+                data = None
+            else:
+                data = type(tables[0]).concatenate(tables, ignore_domains=True)
+                if source_var is not None:
+                    if self.source_column_role == self.ClassRole:
+                        sdata = data.Table.from_numpy(
+                            data.Domain([], source_var),
+                            np.zeros(len(source_ids, 0)), source_ids)
+                        data = type(tables[0].concatenate(sdata, axis=1))
+                    else:
+                        data = data.add_column(
+                            source_var, source_ids.flatten(),
+                            to_metas=self.source_column_role == self.MetaRole)
+        else:
+            if source_var is not None:
+                places = ["class_vars", "attributes", "metas"]
+                domain = add_columns(
+                    domain,
+                    **{places[self.source_column_role]: (source_var,)})
+            tables = [table.transform(domain) for table in tables]
             data = type(tables[0]).concatenate(tables)
-            if source_var:
-                source_ids = np.array(list(flatten(
-                    [i] * len(table) for i, table in enumerate(tables)))).reshape((-1, 1))
+            if source_var is not None:
                 parts = [data.Y, data.X, data.metas]
                 with data.unlocked(parts[self.source_column_role]):
                     data[:, source_var] = source_ids
-        else:
-            data = None
 
         self.Outputs.data.send(data)
 


Test patch:
diff --git a/Orange/data/tests/test_table.py b/Orange/data/tests/test_table.py
--- a/Orange/data/tests/test_table.py
+++ b/Orange/data/tests/test_table.py
@@ -200,6 +200,16 @@ def test_concatenate_names(self):
         joined = Table.concatenate((tab1, tab2, tab3), axis=1)
         self.assertEqual(joined.name, "tab2")
 
+    def test_concatenate_check_domain(self):
+        a, b, c, d, e, f = map(ContinuousVariable, "abcdef")
+        tables = (self._new_table((a, b), (c, ), (d, e), 5),
+                 self._new_table((a, b), (c, ), (d, e), 5),
+                 self._new_table((a, b), (f, ), (d, e), 5))
+
+        with self.assertRaises(ValueError):
+            Table.concatenate(tables, axis=0)
+        Table.concatenate(tables, axis=0, ignore_domains=True)
+
     def test_with_column(self):
         a, b, c, d, e, f, g = map(ContinuousVariable, "abcdefg")
         col = np.arange(9, 14)
@@ -638,7 +648,7 @@ def test_index_by_int(self):
         np.testing.assert_equal(col, data.metas[:, 0])
         self.assertIs(col.base, data.metas)
 
-    def test_index_by_int(self):
+    def test_index_by_str(self):
         data = self.data
 
         col = data.get_column("y")
diff --git a/Orange/widgets/data/tests/test_owconcatenate.py b/Orange/widgets/data/tests/test_owconcatenate.py
--- a/Orange/widgets/data/tests/test_owconcatenate.py
+++ b/Orange/widgets/data/tests/test_owconcatenate.py
@@ -379,6 +379,101 @@ def test_get_unique_vars(self):
 
         self.assertIs(S1, uS1)
 
+    def test_ignore_domain(self):
+        widget = self.widget
+
+        a, b, c, d, e, f, g, h, i = (ContinuousVariable(x) for x in "abcdefghi")
+        j, k, l = (DiscreteVariable(x, values=tuple("xyz")) for x in "jkl")
+        m = DiscreteVariable("m", values=tuple("xyzu"))
+
+        abcj = Table.from_list(Domain([a, b], c, [j]), [[0, 1, 2, 0], [4, 5, 6, 2]])
+        defk = Table.from_list(Domain([d, e], f, [k]), [[3, 4, 5, 1], [6, 7, 8, 2]])
+        ghil = Table.from_list(Domain([g, h], i, [l]), [[7, 8, 9, 0]])
+
+        widget.ignore_names = True
+        widget.append_source_column = True
+        widget.source_column_role = widget.AttributeRole
+        self.send_signal(widget.Inputs.primary_data, abcj)
+        self.send_signal(widget.Inputs.additional_data, defk, 1)
+        self.send_signal(widget.Inputs.additional_data, ghil, 2)
+
+        self.assertTrue(widget.controls.ignore_names.isEnabled())
+        self.assertFalse(widget.controls.ignore_compute_value.isEnabled())
+        self.assertFalse(widget.Error.incompatible_domains.is_shown())
+        out = self.get_output()
+        self.assertEqual(out.domain.attributes[:2], (a, b))
+        self.assertIs(out.domain.class_var, c)
+        self.assertEqual(out.domain.metas, (j, ))
+        np.testing.assert_equal(out.X, [[0, 1, 0],
+                                        [4, 5, 0],
+                                        [3, 4, 1],
+                                        [6, 7, 1],
+                                        [7, 8, 2]])
+        np.testing.assert_equal(out.Y, [2, 6, 5, 8, 9])
+        np.testing.assert_equal(out.metas, [[0], [2], [1], [2], [0]])
+
+        self.send_signal(widget.Inputs.primary_data, None)
+        self.assertFalse(widget.controls.ignore_names.isEnabled())
+        self.assertTrue(widget.controls.ignore_compute_value.isEnabled())
+        self.assertFalse(widget.Error.incompatible_domains.is_shown())
+        self.assertIsNotNone(self.get_output())
+
+        self.send_signal(widget.Inputs.primary_data, abcj)
+        self.assertTrue(widget.controls.ignore_names.isEnabled())
+        self.assertFalse(widget.Error.incompatible_domains.is_shown())
+        out = self.get_output()
+        self.assertEqual(out.domain.attributes[:2], (a, b))
+        self.assertIs(out.domain.class_var, c)
+        self.assertEqual(out.domain.metas, (j, ))
+
+        self.send_signal(widget.Inputs.additional_data, None, 1)
+        self.assertFalse(widget.Error.incompatible_domains.is_shown())
+        out = self.get_output()
+        self.assertEqual(out.domain.attributes[:2], (a, b))
+        self.assertIs(out.domain.class_var, c)
+        self.assertEqual(out.domain.metas, (j, ))
+        np.testing.assert_equal(out.X, [[0, 1, 0],
+                                        [4, 5, 0],
+                                        [7, 8, 1]])
+        np.testing.assert_equal(out.Y, [2, 6, 9])
+        np.testing.assert_equal(out.metas, [[0], [2], [0]])
+
+        self.send_signal(widget.Inputs.additional_data, None, 2)
+        self.assertFalse(widget.Error.incompatible_domains.is_shown())
+        out = self.get_output()
+        self.assertEqual(out.domain.attributes[:2], (a, b))
+        self.assertIs(out.domain.class_var, c)
+        self.assertEqual(out.domain.metas, (j, ))
+        np.testing.assert_equal(out.X, [[0, 1, 0],
+                                        [4, 5, 0]])
+        np.testing.assert_equal(out.Y, [2, 6])
+        np.testing.assert_equal(out.metas, [[0], [2]])
+
+        self.send_signal(widget.Inputs.primary_data, None)
+        self.assertFalse(widget.Error.incompatible_domains.is_shown())
+        self.assertIsNone(self.get_output())
+
+        self.send_signal(widget.Inputs.primary_data, abcj)
+        self.send_signal(widget.Inputs.additional_data, defk, 1)
+        self.assertFalse(widget.Error.incompatible_domains.is_shown())
+        self.assertIsNotNone(self.get_output())
+
+        self.send_signal(widget.Inputs.additional_data,
+                         Table.from_list(Domain([a, b]), [[1, 2]]),
+                         2)
+        self.assertTrue(widget.Error.incompatible_domains.is_shown())
+        self.assertIsNone(self.get_output())
+
+        self.send_signal(widget.Inputs.additional_data,
+                         Table.from_list(Domain([a, b], c, [m]), [[1, 2]]),
+                         2)
+        self.assertTrue(widget.Error.incompatible_domains.is_shown())
+        self.assertIsNone(self.get_output())
+
+        self.send_signal(widget.Inputs.primary_data, None)
+        self.assertFalse(widget.Error.incompatible_domains.is_shown())
+        self.assertIsNotNone(self.get_output())
+
     def test_different_number_decimals(self):
         widget = self.widget