Repository: biolab/orange3

Issue:
Pivot Table is slow when the categories are large
- [ ] **What's your use case?**
I see that orange can [do `pd.groupby` like thing by Pivot Table](https://orange.biolab.si/blog/2019/2019-08-27-pivot-table/), but when the categories are large, the widget is dead, or almost dead. I can do the same groupby using pandas in seconds. 


- [ ] **What's your proposed solution?**
<!-- Be specific, clear, and concise. -->
I don't known why it is so slow in Orange. Maybe because Orange has to show the resuls? If so, could we select to not to see the results, or just see part of the result? Many times, we don't need to see all the results, because we just want to `groupby` and do other tasks in other widgets.









Discussion:
> Many times, we don't need to see all the results, because we just want to groupby and do other tasks in other widgets.

And how many times would that be, considering that I only suggested this trick to you yesterday? :)))))

Any data that is output needs to be computed. I very much doubt that filling the table's cells with data takes more time than computing that data. It would help us if you described your data (or even uploaded/linked to it) and tell us how many groups you create, how, what aggregation functions you compute. Without that, we have nothing to work on.

Also, in general: if you create so many groups that their computation takes time, all or most of them would contain very small samples so any aggregates would be unreliable. So Pivot with a large number of categories is not really useful.
here is the data:  https://drive.google.com/file/d/1mXknFu6VmVojELGLk9z9jhXaotn-DA1U/view?usp=sharing
(I will delete this data once the issue is closed, sorry)

my python code:
```py
# ex is this data
group_ex = ex.groupby('id').agg({'sales': ['mean', 'min', 'max', 'sum']})
group_ex = group_ex.reset_index()
group_ex.columns = ['id', 'mean', 'min', 'max', 'sum']
```
I tried the same thing with pivot table, but the failed. 

some background of this data: this is the daily expense of several people during some months, each row is one person's single expense. I need to get each person's mean, min, max and sum of the expense during this period.

Resolving patch:
diff --git a/Orange/widgets/data/owpivot.py b/Orange/widgets/data/owpivot.py
--- a/Orange/widgets/data/owpivot.py
+++ b/Orange/widgets/data/owpivot.py
@@ -27,7 +27,8 @@
 from Orange.widgets.utils.sql import check_sql_input
 from Orange.widgets.utils.itemmodels import DomainModel
 from Orange.widgets.utils.widgetpreview import WidgetPreview
-from Orange.widgets.utils.state_summary import format_summary_details
+from Orange.widgets.utils.state_summary import format_summary_details, \
+    format_multiple_summaries
 from Orange.widgets.widget import OWWidget, Input, Output, Msg
 
 
@@ -612,7 +613,7 @@ def __set_vertical_headers(self, table):
 
     def _set_values(self, table):
         for i, j in product(range(len(table)), range(len(table[0]))):
-            item = self._create_value_item(str(table[i, j]))
+            item = self._create_value_item(str(table.X[i, j]))
             self.table_model.setItem(i + self._n_leading_rows,
                                      j + self._n_leading_cols, item)
 
@@ -710,13 +711,14 @@ class Warning(OWWidget.Warning):
         cannot_aggregate = Msg("Some aggregations ({}) cannot be computed.")
         renamed_vars = Msg("Some variables have been renamed in some tables"
                            "to avoid duplicates.\n{}")
+        too_many_values = Msg("Selected variable has too many values.")
 
     settingsHandler = DomainContextHandler()
     row_feature = ContextSetting(None)
     col_feature = ContextSetting(None)
     val_feature = ContextSetting(None)
     sel_agg_functions = Setting(set([Pivot.Count]))
-    selection = ContextSetting(set())
+    selection = Setting(set(), schema_only=True)
     auto_commit = Setting(True)
 
     AGGREGATIONS = (Pivot.Count,
@@ -733,10 +735,13 @@ class Warning(OWWidget.Warning):
                     None,
                     Pivot.Majority)
 
+    MAX_VALUES = 100
+
     def __init__(self):
         super().__init__()
         self.data = None  # type: Table
         self.pivot = None  # type: Pivot
+        self.__pending_selection = self.selection  # type: Set
         self._add_control_area_controls()
         self._add_main_area_controls()
 
@@ -761,8 +766,8 @@ def _add_control_area_controls(self):
         gui.rubber(self.controlArea)
         gui.auto_apply(self.controlArea, self, "auto_commit")
 
-        self.info.set_input_summary(self.info.NoInput)
-        self.info.set_output_summary(self.info.NoOutput)
+        self.set_input_summary()
+        self.set_output_summary(None, None, None)
 
     def __add_aggregation_controls(self):
         def new_inbox():
@@ -852,6 +857,7 @@ def __invalidate_filtered(self):
     @check_sql_input
     def set_data(self, data):
         self.closeContext()
+        self.selection = set()
         self.data = data
         self.pivot = None
         self.check_data()
@@ -861,12 +867,7 @@ def set_data(self, data):
 
     def check_data(self):
         self.clear_messages()
-        if not self.data:
-            self.table_view.clear()
-            self.info.set_input_summary(self.info.NoInput)
-        else:
-            self.info.set_input_summary(len(self.data),
-                                        format_summary_details(self.data))
+        self.set_input_summary()
 
     def init_attr_values(self):
         domain = self.data.domain if self.data and len(self.data) else None
@@ -882,31 +883,70 @@ def init_attr_values(self):
                 if domain.variables[0] in model else model[2]
 
     def commit(self):
+        def send_outputs(pivot_table, filtered_data, grouped_data):
+            self.Outputs.grouped_data.send(grouped_data)
+            self.Outputs.pivot_table.send(pivot_table)
+            self.Outputs.filtered_data.send(filtered_data)
+            self.set_output_summary(pivot_table, filtered_data, grouped_data)
+
         self.Warning.renamed_vars.clear()
+        self.Warning.too_many_values.clear()
+        self.Warning.cannot_aggregate.clear()
+        self.Warning.no_col_feature.clear()
+
         if self.pivot is None:
-            self.Warning.no_col_feature.clear()
             if self.no_col_feature:
+                self.table_view.clear()
                 self.Warning.no_col_feature()
+                send_outputs(None, None, None)
                 return
+
+            if self.data:
+                col_var = self.col_feature or self.row_feature
+                col = self.data.get_column_view(col_var)[0].astype(np.float)
+                if len(nanunique(col)) >= self.MAX_VALUES:
+                    self.table_view.clear()
+                    self.Warning.too_many_values()
+                    send_outputs(None, None, None)
+                    return
+
             self.pivot = Pivot(self.data, self.sel_agg_functions,
                                self.row_feature,
                                self.col_feature, self.val_feature)
-        self.Warning.cannot_aggregate.clear()
+
         if self.skipped_aggs:
             self.Warning.cannot_aggregate(self.skipped_aggs)
         self._update_graph()
-        filtered_data = self.get_filtered_data()
-        self.Outputs.grouped_data.send(self.pivot.group_table)
-        self.Outputs.pivot_table.send(self.pivot.pivot_table)
-        self.Outputs.filtered_data.send(filtered_data)
 
-        summary = len(filtered_data) if filtered_data else self.info.NoOutput
-        details = format_summary_details(filtered_data) if filtered_data else ""
-        self.info.set_output_summary(summary, details)
+        send_outputs(self.pivot.pivot_table,
+                     self.get_filtered_data(),
+                     self.pivot.group_table)
 
         if self.pivot.renamed:
             self.Warning.renamed_vars(self.pivot.renamed)
 
+    def set_input_summary(self):
+        summary = len(self.data) if self.data else self.info.NoInput
+        details = format_summary_details(self.data) if self.data else ""
+        self.info.set_input_summary(summary, details)
+
+    def set_output_summary(self, pivot: Table, filtered: Table, grouped: Table):
+        summary, detail, kwargs = self.info.NoOutput, "", {}
+        if pivot or filtered or grouped:
+            n_pivot = len(pivot) if pivot else 0
+            n_filtered = len(filtered) if filtered else 0
+            n_grouped = len(grouped) if grouped else 0
+            summary = f"{self.info.format_number(n_pivot)}, " \
+                      f"{self.info.format_number(n_filtered)}, " \
+                      f"{self.info.format_number(n_grouped)}"
+            detail = format_multiple_summaries([
+                ("Pivot table", pivot),
+                ("Filtered data", filtered),
+                ("Grouped data", grouped)
+            ])
+            kwargs = {"format": Qt.RichText}
+        self.info.set_output_summary(summary, detail, **kwargs)
+
     def _update_graph(self):
         self.table_view.clear()
         if self.pivot.pivot_table:
@@ -914,7 +954,10 @@ def _update_graph(self):
             self.table_view.update_table(col_feature.name,
                                          self.row_feature.name,
                                          *self.pivot.pivot_tables)
-            self.table_view.set_selection(self.selection)
+            selection = self.__pending_selection or self.selection
+            self.table_view.set_selection(selection)
+            self.selection = self.table_view.get_selection()
+            self.__pending_selection = set()
 
     def get_filtered_data(self):
         if not self.data or not self.selection or not self.pivot.pivot_table:


Test patch:
diff --git a/Orange/widgets/data/tests/test_owpivot.py b/Orange/widgets/data/tests/test_owpivot.py
--- a/Orange/widgets/data/tests/test_owpivot.py
+++ b/Orange/widgets/data/tests/test_owpivot.py
@@ -8,7 +8,6 @@
 
 from AnyQt.QtCore import Qt, QPoint
 from AnyQt.QtTest import QTest
-from AnyQt.QtWidgets import QCheckBox
 
 from Orange.data import (Table, Domain, ContinuousVariable as Cv,
                          StringVariable as sv, DiscreteVariable as Dv)
@@ -16,7 +15,8 @@
                                          AggregationFunctionsEnum)
 from Orange.widgets.tests.base import WidgetTest
 from Orange.widgets.tests.utils import simulate
-from Orange.widgets.utils.state_summary import format_summary_details
+from Orange.widgets.utils.state_summary import format_summary_details, \
+    format_multiple_summaries
 
 
 class TestOWPivot(WidgetTest):
@@ -155,9 +155,10 @@ def test_aggregations(self):
         simulate.combobox_activate_item(self.widget.controls.val_feature,
                                         "(None)")
         self.assertTrue(self.widget.Warning.cannot_aggregate.is_shown())
+        # agg: Count, Majority, feature: None, row: Continuous
         simulate.combobox_activate_item(self.widget.controls.row_feature,
                                         self.iris.domain.attributes[1].name)
-        self.assertTrue(self.widget.Warning.cannot_aggregate.is_shown())
+        self.assertFalse(self.widget.Warning.cannot_aggregate.is_shown())
         self.send_signal(self.widget.Inputs.data, None)
         self.assertFalse(self.widget.Warning.cannot_aggregate.is_shown())
 
@@ -247,9 +248,15 @@ def test_summary(self):
                                         self.iris.domain.class_var.name)
         self.widget.table_view.set_selection(set([(11, 0), (11, 1), (12, 0)]))
         self.widget.table_view.selection_changed.emit()
-        output = self.get_output(self.widget.Outputs.filtered_data)
-        output_sum.assert_called_with(len(output),
-                                      format_summary_details(output))
+        filtered = self.get_output(self.widget.Outputs.filtered_data)
+        pivot = self.get_output(self.widget.Outputs.pivot_table)
+        grouped = self.get_output(self.widget.Outputs.grouped_data)
+        output_sum.assert_called_with(
+            f"{len(pivot)}, {len(filtered)}, {len(grouped)}",
+            format_multiple_summaries([("Pivot table", pivot),
+                                       ("Filtered data", filtered),
+                                       ("Grouped data", grouped)]),
+            format=1)
         input_sum.reset_mock()
         output_sum.reset_mock()
         self.send_signal(self.widget.Inputs.data, None)
@@ -268,6 +275,15 @@ def test_renaming_warning(self):
         self.send_signal(self.widget.Inputs.data, self.iris)
         self.assertFalse(self.widget.Warning.renamed_vars.is_shown())
 
+    @patch("Orange.widgets.data.owpivot.OWPivot.MAX_VALUES", 2)
+    def test_max_values(self):
+        self.send_signal(self.widget.Inputs.data, self.iris)
+        self.assertTrue(self.widget.Warning.too_many_values.is_shown())
+
+        self.send_signal(self.widget.Inputs.data, None)
+        self.assertFalse(self.widget.Warning.too_many_values.is_shown())
+
+
 class TestAggregationFunctionsEnum(unittest.TestCase):
     def test_pickle(self):
         self.assertIs(AggregationFunctionsEnum.Sum,