Repository: biolab/orange3

Issue:
Running times in Test & Score
It would be nice to see running times of algorithms -- both training and learning.

A particular problem that we had was that we used 10 different learning algorithms and Test & Score was really slow. We wanted to know which to remove.

However, this information is also interesting by itself.


Discussion:

Resolving patch:
diff --git a/Orange/widgets/evaluate/owtestlearners.py b/Orange/widgets/evaluate/owtestlearners.py
--- a/Orange/widgets/evaluate/owtestlearners.py
+++ b/Orange/widgets/evaluate/owtestlearners.py
@@ -499,6 +499,16 @@ def update_stats_model(self):
             head = QStandardItem(name)
             head.setData(key, Qt.UserRole)
             results = slot.results
+            if results is not None and results.success:
+                train = QStandardItem("{:.3f}".format(results.value.train_time))
+                train.setTextAlignment(Qt.AlignRight)
+                train.setData(key, Qt.UserRole)
+                test = QStandardItem("{:.3f}".format(results.value.test_time))
+                test.setTextAlignment(Qt.AlignRight)
+                test.setData(key, Qt.UserRole)
+                row = [head, train, test]
+            else:
+                row = [head]
             if isinstance(results, Try.Fail):
                 head.setToolTip(str(results.exception))
                 head.setText("{} (error)".format(name))
@@ -512,7 +522,6 @@ def update_stats_model(self):
                                   "{exc.__class__.__name__}: {exc!s}"
                                   .format(name=name, exc=slot.results.exception)
                                   )
-            row = [head]
 
             if class_var is not None and class_var.is_discrete and \
                     target_index is not None:
@@ -532,6 +541,7 @@ def update_stats_model(self):
             if stats is not None:
                 for stat, scorer in zip(stats, self.scorers):
                     item = QStandardItem()
+                    item.setTextAlignment(Qt.AlignRight)
                     if stat.success:
                         item.setText("{:.3f}".format(stat.value[0]))
                     else:
diff --git a/Orange/widgets/evaluate/utils.py b/Orange/widgets/evaluate/utils.py
--- a/Orange/widgets/evaluate/utils.py
+++ b/Orange/widgets/evaluate/utils.py
@@ -162,9 +162,11 @@ def _update_shown_columns(self):
 
     def update_header(self, scorers):
         # Set the correct horizontal header labels on the results_model.
-        self.model.setColumnCount(1 + len(scorers))
+        self.model.setColumnCount(3 + len(scorers))
         self.model.setHorizontalHeaderItem(0, QStandardItem("Model"))
-        for col, score in enumerate(scorers, start=1):
+        self.model.setHorizontalHeaderItem(1, QStandardItem("Train time [s]"))
+        self.model.setHorizontalHeaderItem(2, QStandardItem("Test time [s]"))
+        for col, score in enumerate(scorers, start=3):
             item = QStandardItem(score.name)
             item.setToolTip(score.long_name)
             self.model.setHorizontalHeaderItem(col, item)


Test patch:
diff --git a/Orange/evaluation/testing.py b/Orange/evaluation/testing.py
--- a/Orange/evaluation/testing.py
+++ b/Orange/evaluation/testing.py
@@ -2,6 +2,7 @@
 # pylint: disable=arguments-differ
 from warnings import warn
 from collections import namedtuple
+from time import time
 
 import numpy as np
 
@@ -13,7 +14,8 @@
            "ShuffleSplit", "TestOnTestData", "sample", "CrossValidationFeature"]
 
 _MpResults = namedtuple('_MpResults', ('fold_i', 'learner_i', 'model',
-                                       'failed', 'n_values', 'values', 'probs'))
+                                       'failed', 'n_values', 'values',
+                                       'probs', 'train_time', 'test_time'))
 
 
 def _identity(x):
@@ -23,19 +25,27 @@ def _identity(x):
 def _mp_worker(fold_i, train_data, test_data, learner_i, learner,
                store_models):
     predicted, probs, model, failed = None, None, None, False
+    train_time, test_time = None, None
     try:
         if not train_data or not test_data:
             raise RuntimeError('Test fold is empty')
+        # training
+        t0 = time()
         model = learner(train_data)
+        train_time = time() - t0
+        t0 = time()
+        # testing
         if train_data.domain.has_discrete_class:
             predicted, probs = model(test_data, model.ValueProbs)
         elif train_data.domain.has_continuous_class:
             predicted = model(test_data, model.Value)
+        test_time = time() - t0
     # Different models can fail at any time raising any exception
     except Exception as ex:  # pylint: disable=broad-except
         failed = ex
     return _MpResults(fold_i, learner_i, store_models and model,
-                      failed, len(test_data), predicted, probs)
+                      failed, len(test_data), predicted, probs,
+                      train_time, test_time)
 
 
 class Results:
@@ -72,6 +82,10 @@ class Results:
             corresponding to testing data subsets, that is,
             `row_indices[folds[i]]` contains row indices used in fold i, so
             `data[row_indices[folds[i]]]` is the corresponding testing data
+
+        train_time (np.ndarray): training times of batches
+
+        test_time (np.ndarray): testing times of batches
     """
     def __init__(self, data=None, *,
                  nmethods=None, nrows=None, nclasses=None,
@@ -79,7 +93,8 @@ def __init__(self, data=None, *,
                  row_indices=None, folds=None, score_by_folds=True,
                  learners=None, models=None, failed=None,
                  actual=None, predicted=None, probabilities=None,
-                 store_data=None, store_models=None):
+                 store_data=None, store_models=None,
+                 train_time=None, test_time=None):
         """
         Construct an instance.
 
@@ -138,6 +153,9 @@ def __init__(self, data=None, *,
         self.probabilities = probabilities
         self.failed = failed
 
+        self.train_time = train_time
+        self.test_time = test_time
+
         # Guess the rest -- or check for ambguities
         def set_or_raise(value, exp_values, msg):
             for exp_value in exp_values:
@@ -311,6 +329,8 @@ def split_by_model(self):
             res.actual = self.actual
             res.folds = self.folds
             res.score_by_folds = self.score_by_folds
+            res.test_time = self.test_time[i]
+            res.train_time = self.train_time[i]
 
             res.predicted = self.predicted[(i,), :]
             if getattr(self, "probabilities", None) is not None:
@@ -429,7 +449,10 @@ def __call__(self, data, learners, preprocessor=None, *, callback=None):
             domain=data.domain,
             nrows=len(row_indices), learners=learners,
             row_indices=row_indices, folds=folds, actual=actual,
-            score_by_folds=self.score_by_folds)
+            score_by_folds=self.score_by_folds,
+            train_time=np.zeros((len(learners),)),
+            test_time=np.zeros((len(learners),)))
+
         if self.store_models:
             results.models = np.tile(None, (len(indices), len(learners)))
         self._collect_part_results(results, part_results)
@@ -506,6 +529,8 @@ def _collect_part_results(self, results, part_results):
                 results.models[res.fold_i][res.learner_i] = res.model
 
             results.predicted[res.learner_i][result_slice] = res.values
+            results.train_time[res.learner_i] += res.train_time
+            results.test_time[res.learner_i] += res.test_time
             if res.probs is not None:
                 results.probabilities[res.learner_i][result_slice, :] = \
                     res.probs
@@ -711,7 +736,10 @@ def __call__(self, data, test_data, learners, preprocessor=None,
             row_indices=np.arange(len(test_data)),
             folds=(Ellipsis, ),
             actual=test_data.Y.ravel(),
-            score_by_folds=self.score_by_folds)
+            score_by_folds=self.score_by_folds,
+            train_time=np.zeros((len(learners),)),
+            test_time=np.zeros((len(learners),)))
+
         if self.store_models:
             results.models = np.tile(None, (1, len(learners)))
         self._collect_part_results(results, part_results)
diff --git a/Orange/widgets/evaluate/tests/test_owtestlearners.py b/Orange/widgets/evaluate/tests/test_owtestlearners.py
--- a/Orange/widgets/evaluate/tests/test_owtestlearners.py
+++ b/Orange/widgets/evaluate/tests/test_owtestlearners.py
@@ -225,16 +225,16 @@ def test_target_changing(self):
         self.send_signal(self.widget.Inputs.learner,
                          LogisticRegressionLearner(), 0, wait=5000)
 
-        average_auc = float(model.item(0, 1).text())
+        average_auc = float(model.item(0, 3).text())
 
         simulate.combobox_activate_item(w.controls.class_selection, "Iris-setosa")
-        setosa_auc = float(model.item(0, 1).text())
+        setosa_auc = float(model.item(0, 3).text())
 
         simulate.combobox_activate_item(w.controls.class_selection, "Iris-versicolor")
-        versicolor_auc = float(model.item(0, 1).text())
+        versicolor_auc = float(model.item(0, 3).text())
 
         simulate.combobox_activate_item(w.controls.class_selection, "Iris-virginica")
-        virginica_auc = float(model.item(0, 1).text())
+        virginica_auc = float(model.item(0, 3).text())
 
         self.assertGreater(average_auc, versicolor_auc)
         self.assertGreater(average_auc, virginica_auc)
@@ -287,12 +287,12 @@ def __call__(self, data):
     def _retrieve_scores(self):
         w = self.widget
         model = w.score_table.model
-        auc = model.item(0, 1).text()
+        auc = model.item(0, 3).text()
         auc = float(auc) if auc != "" else None
-        ca = float(model.item(0, 2).text())
-        f1 = float(model.item(0, 3).text())
-        precision = float(model.item(0, 4).text())
-        recall = float(model.item(0, 5).text())
+        ca = float(model.item(0, 4).text())
+        f1 = float(model.item(0, 5).text())
+        precision = float(model.item(0, 6).text())
+        recall = float(model.item(0, 7).text())
         return auc, ca, f1, precision, recall
 
     def _test_scores(self, train_data, test_data, learner, sampling, n_folds):