Repository: biolab/orange3

Issue:
Orange.evaluation.testing.Results should be data class
Class `Orange.evaluation.testing.Results` is typical for early days of Orange 3. It's called `Results` and stores results (predictions and probabilities), but derived classes are `CrossValidation` etc.

`Results` should be a data class (possibly with some smart constructors for merging results, and possibly with some methods for splitting results by methods or folds; functions for this are currently implemented in the Test and Score widget). Sampling methods should have a separate hierarchy.

Since we know that this constructor didn't work properly (until #3858), we can be sure that nobody used it in this way (and it's doubtful that any code outside the core Orange used this class anyway). So it may be safe to refactor this.


Discussion:

Resolving patch:
diff --git a/Orange/evaluation/clustering.py b/Orange/evaluation/clustering.py
--- a/Orange/evaluation/clustering.py
+++ b/Orange/evaluation/clustering.py
@@ -4,7 +4,7 @@
 from sklearn.metrics import silhouette_score, adjusted_mutual_info_score, silhouette_samples
 
 from Orange.data import Table
-from Orange.evaluation.testing import Results
+from Orange.evaluation.testing import Results, Validation
 from Orange.evaluation.scoring import Score
 
 
@@ -12,11 +12,8 @@
 
 
 class ClusteringResults(Results):
-    def __init__(self, store_data=True, **kwargs):
-        super().__init__(store_data=True, **kwargs)
-
     def get_fold(self, fold):
-        results = ClusteringResults()
+        results = Results()
         results.data = self.data
 
         if self.folds is None:
@@ -35,12 +32,13 @@ def get_fold(self, fold):
 class ClusteringScore(Score):
     considers_actual = False
 
+    # pylint: disable=arguments-differ
     def from_predicted(self, results, score_function):
         # Clustering scores from labels
         # This warning filter can be removed in scikit 0.22
         with warnings.catch_warnings():
             warnings.filterwarnings(
-                "ignore", "The behavior of AMI will change in version 0\.22.*")
+                "ignore", "The behavior of AMI will change in version 0\\.22.*")
             if self.considers_actual:
                 return np.fromiter(
                     (score_function(results.actual.flatten(),
@@ -70,48 +68,39 @@ def compute_score(self, results):
         return self.from_predicted(results, adjusted_mutual_info_score)
 
 
-class ClusteringEvaluation(ClusteringResults):
+# Class overrides fit and doesn't need to define the abstract get_indices
+# pylint: disable=abstract-method
+class ClusteringEvaluation(Validation):
     """
     Clustering evaluation.
 
-    If the constructor is given the data and a list of learning algorithms, it
-    runs clustering and returns an instance of `Results` containing the
-    predicted clustering labels.
-
     .. attribute:: k
         The number of runs.
 
     """
-    def __init__(self, data, learners, k=1,
-                 store_models=False):
-        super().__init__(data=data, nmethods=len(learners), store_data=True,
-                         store_models=store_models, predicted=None)
-
+    def __init__(self, k=1, store_data=False, store_models=False):
+        super().__init__(store_data=store_data, store_models=store_models)
         self.k = k
-        Y = data.Y.copy().flatten()
-
-        self.predicted = np.empty((len(learners), self.k, len(data)))
-        self.folds = range(k)
-        self.row_indices = np.arange(len(data))
-        self.actual = data.Y.flatten() if hasattr(data, "Y") else None
 
+    def __call__(self, data, learners, preprocessor=None, *, callback=None):
+        res = ClusteringResults()
+        res.data = data
+        res.predicted = np.empty((len(learners), self.k, len(data)))
+        res.folds = range(self.k)
+        res.row_indices = np.arange(len(data))
+        res.actual = data.Y.flatten() if hasattr(data, "Y") else None
         if self.store_models:
-            self.models = []
+            res.models = np.tile(None, (self.k, len(learners)))
 
         for k in range(self.k):
-
-            if self.store_models:
-                fold_models = []
-                self.models.append(fold_models)
-
             for i, learner in enumerate(learners):
                 model = learner(data)
                 if self.store_models:
-                    fold_models.append(model)
-
+                    res.models[k, i] = model
                 labels = model(data)
-                self.predicted[i, k, :] = labels.X.flatten()
+                res.predicted[i, k, :] = labels.X.flatten()
 
+        return res
 
 
 def graph_silhouette(X, y, xlim=None, colors=None, figsize=None, filename=None):
diff --git a/Orange/widgets/evaluate/owtestlearners.py b/Orange/widgets/evaluate/owtestlearners.py
--- a/Orange/widgets/evaluate/owtestlearners.py
+++ b/Orange/widgets/evaluate/owtestlearners.py
@@ -88,7 +88,7 @@ def __repr__(self):
             return "{}({!r})".format(self.__class__.__qualname__,
                                      self.exception)
 
-        def map(self, fn):
+        def map(self, _fn):
             return self
 
     def __new__(cls, f, *args, **kwargs):
@@ -283,7 +283,8 @@ def __init__(self):
         box = gui.vBox(self.mainArea, "Evaluation Results")
         box.layout().addWidget(self.score_table.view)
 
-    def sizeHint(self):
+    @staticmethod
+    def sizeHint():
         return QSize(780, 1)
 
     def _update_controls(self):
@@ -333,7 +334,7 @@ def set_train_data(self, data):
         self.Error.too_many_classes.clear()
         self.Error.no_class_values.clear()
         self.Error.only_one_class_var_value.clear()
-        if data is not None and not len(data):
+        if data is not None and not data:
             self.Error.train_data_empty()
             data = None
         if data:
@@ -392,7 +393,7 @@ def set_test_data(self, data):
         """
         self.Information.test_data_sampled.clear()
         self.Error.test_data_empty.clear()
-        if data is not None and not len(data):
+        if data is not None and not data:
             self.Error.test_data_empty()
             data = None
         if data and not data.domain.class_var:
@@ -721,10 +722,6 @@ def __update(self):
             self.Warning.test_data_unused()
 
         rstate = 42
-        common_args = dict(
-            store_data=True,
-            preprocessor=self.preprocessor,
-        )
         # items in need of an update
         items = [(key, slot) for key, slot in self.learners.items()
                  if slot.results is None]
@@ -736,45 +733,36 @@ def __update(self):
         # learners bellow)
         learners_c = [copy.deepcopy(learner) for learner in learners]
 
-        if self.resampling == OWTestLearners.KFold:
-            folds = self.NFolds[self.n_folds]
-            test_f = partial(
-                Orange.evaluation.CrossValidation,
-                self.data, learners_c, k=folds,
-                random_state=rstate, **common_args)
-        elif self.resampling == OWTestLearners.FeatureFold:
-            test_f = partial(
-                Orange.evaluation.CrossValidationFeature,
-                self.data, learners_c, self.fold_feature,
-                **common_args
-            )
-        elif self.resampling == OWTestLearners.LeaveOneOut:
-            test_f = partial(
-                Orange.evaluation.LeaveOneOut,
-                self.data, learners_c, **common_args
-            )
-        elif self.resampling == OWTestLearners.ShuffleSplit:
-            train_size = self.SampleSizes[self.sample_size] / 100
-            test_f = partial(
-                Orange.evaluation.ShuffleSplit,
-                self.data, learners_c,
-                n_resamples=self.NRepeats[self.n_repeats],
-                train_size=train_size, test_size=None,
-                stratified=self.shuffle_stratified,
-                random_state=rstate, **common_args
-            )
-        elif self.resampling == OWTestLearners.TestOnTrain:
-            test_f = partial(
-                Orange.evaluation.TestOnTrainingData,
-                self.data, learners_c, **common_args
-            )
-        elif self.resampling == OWTestLearners.TestOnTest:
+        if self.resampling == OWTestLearners.TestOnTest:
             test_f = partial(
-                Orange.evaluation.TestOnTestData,
-                self.data, self.test_data, learners_c, **common_args
+                Orange.evaluation.TestOnTestData(store_data=True),
+                self.data, self.test_data, learners_c, self.preprocessor
             )
         else:
-            assert False, "self.resampling %s" % self.resampling
+            if self.resampling == OWTestLearners.KFold:
+                sampler = Orange.evaluation.CrossValidation(
+                    k=self.NFolds[self.n_folds],
+                    random_state=rstate)
+            elif self.resampling == OWTestLearners.FeatureFold:
+                sampler = Orange.evaluation.CrossValidationFeature(
+                    feature=self.fold_feature)
+            elif self.resampling == OWTestLearners.LeaveOneOut:
+                sampler = Orange.evaluation.LeaveOneOut()
+            elif self.resampling == OWTestLearners.ShuffleSplit:
+                sampler = Orange.evaluation.ShuffleSplit(
+                    n_resamples=self.NRepeats[self.n_repeats],
+                    train_size=self.SampleSizes[self.sample_size] / 100,
+                    test_size=None,
+                    stratified=self.shuffle_stratified,
+                    random_state=rstate)
+            elif self.resampling == OWTestLearners.TestOnTrain:
+                sampler = Orange.evaluation.TestOnTrainingData()
+            else:
+                assert False, "self.resampling %s" % self.resampling
+
+            sampler.store_data = True
+            test_f = partial(
+                sampler, self.data, learners_c, self.preprocessor)
 
         def replace_learners(evalfunc, *args, **kwargs):
             res = evalfunc(*args, **kwargs)
@@ -901,7 +889,6 @@ class UserInterrupt(BaseException):
     """
     A BaseException subclass used for cooperative task/thread cancellation
     """
-    pass
 
 
 def results_add_by_model(x, y):
@@ -1004,21 +991,20 @@ def cancel(self):
 
 if __name__ == "__main__":  # pragma: no cover
     filename = "iris"
-    data = Table(filename)
-    class_var = data.domain.class_var
-    if class_var.is_discrete:
-        learners = [lambda data: 1 / 0,
-                    Orange.classification.LogisticRegressionLearner(),
-                    Orange.classification.MajorityLearner(),
-                    Orange.classification.NaiveBayesLearner()]
+    preview_data = Table(filename)
+    if preview_data.domain.class_var.is_discrete:
+        prev_learners = [lambda data: 1 / 0,
+                         Orange.classification.LogisticRegressionLearner(),
+                         Orange.classification.MajorityLearner(),
+                         Orange.classification.NaiveBayesLearner()]
     else:
-        learners = [lambda data: 1 / 0,
-                    Orange.regression.MeanLearner(),
-                    Orange.regression.KNNRegressionLearner(),
-                    Orange.regression.RidgeRegressionLearner()]
+        prev_learners = [lambda data: 1 / 0,
+                         Orange.regression.MeanLearner(),
+                         Orange.regression.KNNRegressionLearner(),
+                         Orange.regression.RidgeRegressionLearner()]
 
     WidgetPreview(OWTestLearners).run(
-        set_train_data=data,
-        set_test_data=data,
-        set_learner=[(learner, i) for i, learner in enumerate(learners)]
+        set_train_data=preview_data,
+        set_test_data=preview_data,
+        set_learner=[(learner, i) for i, learner in enumerate(prev_learners)]
     )


Test patch:
diff --git a/Orange/evaluation/testing.py b/Orange/evaluation/testing.py
--- a/Orange/evaluation/testing.py
+++ b/Orange/evaluation/testing.py
@@ -1,3 +1,6 @@
+# __new__ methods have different arguments
+# pylint: disable=arguments-differ
+from warnings import warn
 from collections import namedtuple
 
 import numpy as np
@@ -21,7 +24,7 @@ def _mp_worker(fold_i, train_data, test_data, learner_i, learner,
                store_models):
     predicted, probs, model, failed = None, None, None, False
     try:
-        if len(train_data) == 0 or len(test_data) == 0:
+        if not train_data or not test_data:
             raise RuntimeError('Test fold is empty')
         model = learner(train_data)
         if train_data.domain.has_discrete_class:
@@ -40,111 +43,102 @@ class Results:
     Class for storing predictions in model testing.
 
     Attributes:
-        data (Optional[Table]):  Data used for testing. When data is stored,
-            this is typically not a copy but a reference.
+        data (Optional[Table]):  Data used for testing.
 
         models (Optional[List[Model]]): A list of induced models.
 
         row_indices (np.ndarray): Indices of rows in `data` that were used in
             testing, stored as a numpy vector of length `nrows`.
             Values of `actual[i]`, `predicted[i]` and `probabilities[i]` refer
-            to the target value of instance `data[row_indices[i]]`.
+            to the target value of instance, that is, the i-th test instance
+            is `data[row_indices[i]]`, its actual class is `actual[i]`, and
+            the prediction by m-th method is `predicted[m, i]`.
 
-        nrows (int): The number of test instances (including duplicates).
+        nrows (int): The number of test instances (including duplicates);
+            `nrows` equals the length of `row_indices` and `actual`, and the
+            second dimension of `predicted` and `probabilities`.
 
-        actual (np.ndarray): Actual values of target variable;
-            a numpy vector of length `nrows` and of the same type as `data`
-            (or `np.float32` if the type of data cannot be determined).
+        actual (np.ndarray): true values of target variable in a vector of
+            length `nrows`.
 
-        predicted (np.ndarray): Predicted values of target variable;
-            a numpy array of shape (number-of-methods, `nrows`) and
-            of the same type as `data` (or `np.float32` if the type of data
-            cannot be determined).
+        predicted (np.ndarray): predicted values of target variable in an array
+            of shape (number-of-methods, `nrows`)
 
-        probabilities (Optional[np.ndarray]): Predicted probabilities
-            (for discrete target variables);
-            a numpy array of shape (number-of-methods, `nrows`, number-of-classes)
-            of type `np.float32`.
+        probabilities (Optional[np.ndarray]): predicted probabilities
+            (for discrete target variables) in an array of shape
+            (number-of-methods, `nrows`, number-of-classes)
 
-        folds (List[Slice or List[int]]): A list of indices (or slice objects)
-            corresponding to rows of each fold.
+        folds (List[Slice or List[int]]): a list of indices (or slice objects)
+            corresponding to testing data subsets, that is,
+            `row_indices[folds[i]]` contains row indices used in fold i, so
+            `data[row_indices[folds[i]]]` is the corresponding testing data
     """
-    score_by_folds = True
-    # noinspection PyBroadException
-    # noinspection PyNoneFunctionAssignment
-    def __init__(self, data=None, nmethods=None, *, learners=None,
-                 train_data=None, nrows=None, nclasses=None,
-                 store_data=False, store_models=False,
-                 domain=None, actual=None, row_indices=None,
-                 predicted=None, probabilities=None,
-                 preprocessor=None, callback=None, n_jobs=1):
+    def __init__(self, data=None, *,
+                 nmethods=None, nrows=None, nclasses=None,
+                 domain=None,
+                 row_indices=None, folds=None, score_by_folds=True,
+                 learners=None, models=None, failed=None,
+                 actual=None, predicted=None, probabilities=None,
+                 store_data=None, store_models=None):
         """
-        Construct an instance with default values: `None` for :obj:`data` and
-        :obj:`models`.
-
-        If the number of rows and/or the number of classes is not given, it is
-        inferred from :obj:`data`, if provided. The data type for
-        :obj:`actual` and :obj:`predicted` is determined from the data; if the
-        latter cannot be find, `np.float32` is used.
-
-        Attribute :obj:`actual` and :obj:`row_indices` are constructed as empty
-        (uninitialized) arrays of the appropriate size, if the number of rows
-        is known. Attribute :obj:`predicted` is constructed if the number of
-        rows and of methods is given; :obj:`probabilities` also requires
-        knowing the number of classes.
-
-        :param data: Data or domain
-        :type data: Orange.data.Table or Orange.data.Domain
-        :param nmethods: The number of methods that will be tested
-        :type nmethods: int
-        :param nrows: The number of test instances (including duplicates)
-        :type nrows: int
-        :param nclasses: The number of class values
-        :type nclasses: int
-        :param store_data: A flag that tells whether to store the data;
-            this argument can be given only as keyword argument
-        :type store_data: bool
-        :param store_models: A flag that tells whether to store the models;
-            this argument can be given only as keyword argument
-        :type store_models: bool
-        :param preprocessor: Preprocessor for training data
-        :type preprocessor: Orange.preprocess.Preprocess
-        :param callback: Function for reporting back the progress as a value
-            between 0 and 1
-        :type callback: callable
-        :param n_jobs:
-            Ignored.
-        :type n_jobs: int
+        Construct an instance.
+
+        The constructor stores the given data, and creates empty arrays
+        `actual`, `predicted` and `probabilities` if ther are not given but
+        sufficient data is provided to deduct their shapes.
+
+        The function
+
+        - set any attributes specified directly through arguments.
+        - infers the number of methods, rows and classes from other data
+           and/or check their overall consistency.
+        - Prepare empty arrays `actual`, `predicted`, `probabilities` and
+          `failed` if the are not given. If not enough data is available,
+          the corresponding arrays are `None`.
+
+        Args:
+            data (Orange.data.Table): stored data from which test was sampled
+            nmethods (int): number of methods; can be inferred (or must match)
+                the size of `learners`, `models`, `failed`, `predicted` and
+                `probabilities`
+            nrows (int): number of data instances; can be inferred (or must
+                match) `data`, `row_indices`, `actual`, `predicted` and
+                `probabilities`
+            nclasses (int): number of class values (`None` if continuous); can
+                be inferred (or must match) from `domain.class_var` or
+                `probabilities`
+            domain (Orange.data.Domain): data domain; can be inferred (or must)
+                match `data.domain`
+            row_indices (np.ndarray): see class documentation
+            folds (np.ndarray): see class documentation
+            score_by_folds (np.ndarray): see class documentation
+            learners (np.ndarray): see class documentation
+            models (np.ndarray): see class documentation
+            failed (list of str): see class documentation
+            actual (np.ndarray): see class documentation
+            predicted (np.ndarray): see class documentation
+            probabilities (np.ndarray): see class documentation
+            store_data (bool): ignored; kept for backward compatibility
+            store_models (bool): ignored; kept for backward compatibility
         """
-        self.store_data = store_data
-        self.store_models = store_models
-        self.dtype = np.float32
-        self.n_jobs = 1
 
-        self.models = None
-        self.folds = None
-        self.indices = None
+        # Set given data directly from arguments
+        self.data = data
+        self.domain = domain
 
         self.row_indices = row_indices
-        self.preprocessor = preprocessor or _identity
-        self._callback = callback or _identity
-        self.learners = learners
-        if learners:
-            nmethods = len(learners)
-
-        if nmethods is not None:
-            self.failed = [False] * nmethods
+        self.folds = folds
+        self.score_by_folds = score_by_folds
 
-        if data is not None:
-            self.data = data if self.store_data else None
-            self.domain = data.domain
-            self.dtype = getattr(data.Y, 'dtype', self.dtype)
+        self.learners = learners
+        self.models = models
 
-        if learners:
-            train_data = train_data or data
-            self.fit(train_data, data)
-            return
+        self.actual = actual
+        self.predicted = predicted
+        self.probabilities = probabilities
+        self.failed = failed
 
+        # Guess the rest -- or check for ambguities
         def set_or_raise(value, exp_values, msg):
             for exp_value in exp_values:
                 if exp_value is False:
@@ -154,59 +148,53 @@ def set_or_raise(value, exp_values, msg):
                 elif value != exp_value:
                     raise ValueError(msg)
             return value
-
         domain = self.domain = set_or_raise(
             domain, [data is not None and data.domain],
             "mismatching domain")
         self.nrows = nrows = set_or_raise(
-            nrows, [data is not None and len(data),
-                    actual is not None and len(actual),
+            nrows, [actual is not None and len(actual),
                     row_indices is not None and len(row_indices),
                     predicted is not None and predicted.shape[1],
                     probabilities is not None and probabilities.shape[1]],
             "mismatching number of rows")
+        if domain is not None and domain.has_continuous_class:
+            if nclasses is not None:
+                raise ValueError(
+                    "regression results cannot have non-None 'nclasses'")
+            if probabilities is not None:
+                raise ValueError(
+                    "regression results cannot have 'probabilities'")
         nclasses = set_or_raise(
-            nclasses, [len(domain.class_var.values)
-                       if domain is not None and domain.has_discrete_class
-                       else None,
+            nclasses, [domain is not None and domain.has_discrete_class and
+                       len(domain.class_var.values),
                        probabilities is not None and probabilities.shape[2]],
             "mismatching number of class values")
-        if nclasses is None and probabilities is not None:
-            raise ValueError("regression results cannot have 'probabilities'")
         nmethods = set_or_raise(
-            nmethods, [predicted is not None and predicted.shape[0],
+            nmethods, [learners is not None and len(learners),
+                       models is not None and len(models),
+                       failed is not None and len(failed),
+                       predicted is not None and predicted.shape[0],
                        probabilities is not None and probabilities.shape[0]],
             "mismatching number of methods")
 
-        if actual is not None:
-            self.actual = actual
-        elif nrows is not None:
-            self.actual = np.empty(nrows, dtype=self.dtype)
+        # Prepare empty arrays
+        if actual is None \
+                and nrows is not None:
+            self.actual = np.empty(nrows)
 
-        if predicted is not None:
-            self.predicted = predicted
-        elif nmethods is not None and nrows is not None:
-            self.predicted = np.empty((nmethods, nrows), dtype=self.dtype)
+        if predicted is None \
+                and nmethods is not None and nrows is not None:
+            self.predicted = np.empty((nmethods, nrows))
 
-        if probabilities is not None:
-            self.probabilities = probabilities
-        elif nmethods is not None and nrows is not None and \
-                nclasses is not None:
+        if probabilities is None \
+                and nmethods is not None and nrows is not None \
+                and nclasses is not None:
             self.probabilities = \
-                np.empty((nmethods, nrows, nclasses), dtype=np.float32)
+                np.empty((nmethods, nrows, nclasses))
 
-    def _prepare_arrays(self, data):
-        """Initialize some mandatory arrays for results"""
-        nmethods = len(self.learners)
-        self.nrows = len(self.row_indices)
-        if self.store_models:
-            self.models = np.tile(None, (len(self.indices), nmethods))
-        # Initialize `predicted` and `probabilities` (only for discrete classes)
-        self.predicted = np.empty((nmethods, self.nrows), dtype=self.dtype)
-        if data.domain.has_discrete_class:
-            nclasses = len(data.domain.class_var.values)
-            self.probabilities = np.empty((nmethods, self.nrows, nclasses),
-                                          dtype=np.float32)
+        if failed is None \
+                and nmethods is not None:
+            self.failed = [False] * nmethods
 
     def get_fold(self, fold):
         results = Results()
@@ -228,20 +216,26 @@ def get_fold(self, fold):
 
         return results
 
-    def get_augmented_data(self, model_names, include_attrs=True, include_predictions=True,
+    def get_augmented_data(self, model_names,
+                           include_attrs=True, include_predictions=True,
                            include_probabilities=True):
         """
-        Return the data, augmented with predictions, probabilities (if the task is classification)
-        and folds info. Predictions, probabilities and folds are inserted as meta attributes.
+        Return the test data table augmented with meta attributes containing
+        predictions, probabilities (if the task is classification) and fold
+        indices.
 
         Args:
-            model_names (list): A list of strings containing learners' names.
-            include_attrs (bool): Flag that tells whether to include original attributes.
-            include_predictions (bool): Flag that tells whether to include predictions.
-            include_probabilities (bool): Flag that tells whether to include probabilities.
+            model_names (list of str): names of models
+            include_attrs (bool):
+                if set to `False`, original attributes are removed
+            include_predictions (bool):
+                if set to `False`, predictions are not added
+            include_probabilities (bool):
+                if set to `False`, probabilities are not added
 
         Returns:
-            Orange.data.Table: Data augmented with predictions, (probabilities) and (fold).
+            augmented_data (Orange.data.Table):
+                data augmented with predictions, probabilities and fold indices
 
         """
         assert self.predicted.shape[0] == len(model_names)
@@ -256,30 +250,33 @@ def get_augmented_data(self, model_names, include_attrs=True, include_prediction
         if classification:
             # predictions
             if include_predictions:
-                new_meta_attr.extend(DiscreteVariable(name=name, values=class_var.values)
-                                     for name in model_names)
+                new_meta_attr += (
+                    DiscreteVariable(name=name, values=class_var.values)
+                    for name in model_names)
                 new_meta_vals = np.hstack((new_meta_vals, self.predicted.T))
 
             # probabilities
             if include_probabilities:
                 for name in model_names:
-                    new_meta_attr.extend(ContinuousVariable(name="%s (%s)" % (name, value))
-                                         for value in class_var.values)
+                    new_meta_attr += (
+                        ContinuousVariable(name=f"{name} ({value})")
+                        for value in class_var.values)
 
                 for i in self.probabilities:
                     new_meta_vals = np.hstack((new_meta_vals, i))
 
         elif include_predictions:
             # regression
-            new_meta_attr.extend(ContinuousVariable(name=name)
-                                 for name in model_names)
+            new_meta_attr += (ContinuousVariable(name=name)
+                              for name in model_names)
             new_meta_vals = np.hstack((new_meta_vals, self.predicted.T))
 
         # add fold info
         if self.folds is not None:
             new_meta_attr.append(
-                DiscreteVariable(name="Fold",
-                                 values=[str(i+1) for i, _ in enumerate(self.folds)]))
+                DiscreteVariable(
+                    name="Fold",
+                    values=[str(i + 1) for i in range(len(self.folds))]))
             fold = np.empty((len(data), 1))
             for i, s in enumerate(self.folds):
                 fold[s, 0] = i
@@ -296,89 +293,12 @@ def get_augmented_data(self, model_names, include_attrs=True, include_prediction
         predictions.name = data.name
         return predictions
 
-    def fit(self, train_data, test_data=None):
-        """Fits `self.learners` using folds sampled from the provided data.
-
-        Parameters
-        ----------
-        train_data : Table
-            table to sample train folds
-        test_data : Optional[Table]
-            tap to sample test folds of None then `train_data` will be used
-        """
-        test_data = test_data or train_data
-        self.setup_indices(train_data, test_data)
-        self.prepare_arrays(test_data)
-        self._prepare_arrays(test_data)
-
-        n_callbacks = len(self.learners) * len(self.indices)
-
-        data_splits = (
-            (fold_i, self.preprocessor(train_data[train_i]), test_data[test_i])
-            for fold_i, (train_i, test_i) in enumerate(self.indices))
-        args_iter = (
-            (fold_i, train_data, test_data, learner_i, learner,
-             self.store_models)
-            for (fold_i, train_data, test_data) in data_splits
-            for (learner_i, learner) in enumerate(self.learners))
-
-        results = []
-        parts = np.linspace(.0, .99, n_callbacks + 1)[1:]
-        for progress, part in zip(parts, args_iter):
-            results.append(_mp_worker(*(part + ())))
-            self._callback(progress)
-
-        results = sorted(results)
-
-        ptr, prev_fold_i, prev_n_values = 0, 0, 0
-        for res in results:
-            if res.fold_i != prev_fold_i:
-                ptr += prev_n_values
-                prev_fold_i = res.fold_i
-            result_slice = slice(ptr, ptr + res.n_values)
-            prev_n_values = res.n_values
-
-            if res.failed:
-                self.failed[res.learner_i] = res.failed
-                continue
-
-            if self.store_models:
-                self.models[res.fold_i][res.learner_i] = res.model
-
-            self.predicted[res.learner_i][result_slice] = res.values
-            if train_data.domain.has_discrete_class:
-                self.probabilities[res.learner_i][result_slice, :] = res.probs
-
-        self._callback(1)
-        return self
-
-    def prepare_arrays(self, test_data):
-        """Initialize arrays that will be used by `fit` method.
-        """
-        self.folds = []
-        row_indices = []
-
-        ptr = 0
-        for _, test in self.indices:
-            self.folds.append(slice(ptr, ptr + len(test)))
-            row_indices.append(test)
-            ptr += len(test)
-
-        self.row_indices = np.concatenate(row_indices, axis=0)
-        self.actual = test_data[self.row_indices].Y.ravel()
-
-    def setup_indices(self, train_data, test_data):
-        """Initializes `self.indices` with iterable objects with slices
-        (or indices) for each fold.
-
-        Args:
-            train_data (Table): train table
-            test_data (Table): test table
+    def split_by_model(self):
         """
-        raise NotImplementedError()
+        Split evaluation results by models.
 
-    def split_by_model(self):
-        """Split evaluation results by models
+        The method generates instances of `Results` containing data for single
+        models
         """
         data = self.data
         nmethods = len(self.predicted)
@@ -403,174 +323,419 @@ def split_by_model(self):
             yield res
 
 
-class CrossValidation(Results):
+class Validation:
+    """
+    Base class for different testing schemata such as cross validation and
+    testing on separate data set.
+
+    If `data` is some data table and `learners` is a list of learning
+    algorithms. This will run 5-fold cross validation and store the results
+    in `res`.
+
+        cv = CrossValidation(k=5)
+        res = cv(data, learners)
+
+    If constructor was given data and learning algorithms (as in
+    `res = CrossValidation(data, learners, k=5)`, it used to automagically
+    call the instance after constructing it and return `Results` instead
+    of an instance of `Validation`. This functionality
+    is deprecated and will be removed in the future.
+
+    Attributes:
+        store_data (bool): a flag defining whether the data is stored
+        store_models (bool): a flag defining whether the models are stored
     """
-    K-fold cross validation.
+    score_by_folds = False
+
+    def __new__(cls,
+                data=None, learners=None, preprocessor=None, test_data=None,
+                *, callback=None, store_data=False, store_models=False,
+                n_jobs=None, **kwargs):
+        self = super().__new__(cls)
+
+        if (learners is None) != (data is None):
+            raise ValueError(
+                "learners and train_data must both be present or not")
+        if learners is None:
+            if preprocessor is not None:
+                raise ValueError("preprocessor cannot be given if learners "
+                                 "and train_data are omitted")
+            if callback is not None:
+                raise ValueError("callback cannot be given if learners "
+                                 "and train_data are omitted")
+            return self
+
+        warn("calling Validation's constructor with data and learners"
+             "is deprecated;\nconstruct an instance and call it",
+             DeprecationWarning, stacklevel=2)
+
+        # Explicitly call __init__ because Python won't
+        self.__init__(store_data=store_data, store_models=store_models,
+                      **kwargs)
+        if test_data is not None:
+            test_data_kwargs = {"test_data": test_data}
+        else:
+            test_data_kwargs = {}
+        return self(data, learners=learners, preprocessor=preprocessor,
+                    callback=callback, **test_data_kwargs)
+
+    # Note: this will be called only if __new__ doesn't have data and learners
+    def __init__(self, *, store_data=False, store_models=False):
+        self.store_data = store_data
+        self.store_models = store_models
+
+    def fit(self, *args, **kwargs):
+        warn("Validation.fit is deprecated; use the call operator",
+             DeprecationWarning)
+        return self(*args, **kwargs)
+
+    def __call__(self, data, learners, preprocessor=None, *, callback=None):
+        """
+        Args:
+            data (Orange.data.Table): data to be used (usually split) into
+                training and testing
+            learners (list of Orange.Learner): a list of learning algorithms
+            preprocessor (Orange.preprocess.Preprocess): preprocessor applied
+                on training data
+            callback (Callable): a function called to notify about the progress
+
+        Returns:
+            results (Result): results of testing
+        """
+        if preprocessor is None:
+            preprocessor = _identity
+        if callback is None:
+            callback = _identity
+        indices = self.get_indices(data)
+        folds, row_indices, actual = self.prepare_arrays(data, indices)
+
+        data_splits = (
+            (fold_i, preprocessor(data[train_i]), data[test_i])
+            for fold_i, (train_i, test_i) in enumerate(indices))
+        args_iter = (
+            (fold_i, data, test_data, learner_i, learner, self.store_models)
+            for (fold_i, data, test_data) in data_splits
+            for (learner_i, learner) in enumerate(learners))
 
-    If the constructor is given the data and a list of learning algorithms, it
-    runs cross validation and returns an instance of `Results` containing the
-    predicted values and probabilities.
+        part_results = []
+        parts = np.linspace(.0, .99, len(learners) * len(indices) + 1)[1:]
+        for progress, part in zip(parts, args_iter):
+            part_results.append(_mp_worker(*(part + ())))
+            callback(progress)
+        callback(1)
+
+        results = Results(
+            data=data if self.store_data else None,
+            domain=data.domain,
+            nrows=len(row_indices), learners=learners,
+            row_indices=row_indices, folds=folds, actual=actual,
+            score_by_folds=self.score_by_folds)
+        if self.store_models:
+            results.models = np.tile(None, (len(indices), len(learners)))
+        self._collect_part_results(results, part_results)
+        return results
+
+    @classmethod
+    def prepare_arrays(cls, data, indices):
+        """Prepare `folds`, `row_indices` and `actual`.
+
+        The method is used by `__call__`. While functional, it may be
+        overriden in subclasses for speed-ups.
+
+        Args:
+            data (Orange.data.Table): data use for testing
+            indices (list of vectors):
+                indices of data instances in each test sample
+
+        Returns:
+            folds: (np.ndarray): see class documentation
+            row_indices: (np.ndarray): see class documentation
+            actual: (np.ndarray): see class documentation
+        """
+        folds = []
+        row_indices = []
+
+        ptr = 0
+        for _, test in indices:
+            folds.append(slice(ptr, ptr + len(test)))
+            row_indices.append(test)
+            ptr += len(test)
+
+        row_indices = np.concatenate(row_indices, axis=0)
+        actual = data[row_indices].Y.ravel()
+        return folds, row_indices, actual
+
+    @staticmethod
+    def get_indices(data):
+        """
+        Return a list of arrays of indices of test data instance
+
+        For example, in k-fold CV, the result is a list with `k` elements,
+        each containing approximately `len(data) / k` nonoverlapping indices
+        into `data`.
 
-    .. attribute:: k
+        This method is abstract and must be implemented in derived classes
+        unless they provide their own implementation of the `__call__`
+        method.
 
-        The number of folds.
+        Args:
+            data (Orange.data.Table): test data
+
+        Returns:
+            indices (list of np.ndarray):
+                a list of arrays of indices into `data`
+        """
+        raise NotImplementedError()
+
+    def _collect_part_results(self, results, part_results):
+        part_results = sorted(part_results)
+
+        ptr, prev_fold_i, prev_n_values = 0, 0, 0
+        for res in part_results:
+            if res.fold_i != prev_fold_i:
+                ptr += prev_n_values
+                prev_fold_i = res.fold_i
+            result_slice = slice(ptr, ptr + res.n_values)
+            prev_n_values = res.n_values
+
+            if res.failed:
+                results.failed[res.learner_i] = res.failed
+                continue
+
+            if self.store_models:
+                results.models[res.fold_i][res.learner_i] = res.model
 
-    .. attribute:: random_state
+            results.predicted[res.learner_i][result_slice] = res.values
+            if res.probs is not None:
+                results.probabilities[res.learner_i][result_slice, :] = \
+                    res.probs
 
+
+class CrossValidation(Validation):
     """
-    def __init__(self, data, learners, k=10, stratified=True, random_state=0, store_data=False,
-                 store_models=False, preprocessor=None, callback=None, warnings=None,
-                 n_jobs=1):
-        self.k = int(k)
+    K-fold cross validation
+
+    Attributes:
+        k (int): number of folds (default: 10)
+        random_state (int):
+            seed for random number generator (default: 0). If set to `None`,
+            a different seed is used each time
+        stratified (bool):
+            flag deciding whether to perform stratified cross-validation.
+            If `True` but the class sizes don't allow it, it uses non-stratified
+            validataion and adds a list `warning` with a warning message(s) to
+            the `Result`.
+    """
+    # TODO: list `warning` contains just repetitions of the same message
+    #       replace with a flag in `Results`?
+    def __init__(self, k=10, stratified=True, random_state=0,
+                 store_data=False, store_models=False, warnings=None):
+        super().__init__(store_data=store_data, store_models=store_models)
+        self.k = k
         self.stratified = stratified
         self.random_state = random_state
-        if warnings is None:
-            self.warnings = []
-        else:
-            self.warnings = warnings
+        self.warnings = [] if warnings is None else warnings
 
-        super().__init__(data, learners=learners, store_data=store_data,
-                         store_models=store_models, preprocessor=preprocessor,
-                         callback=callback, n_jobs=n_jobs)
-
-    def setup_indices(self, train_data, test_data):
-        self.indices = None
-        if self.stratified and test_data.domain.has_discrete_class:
+    def get_indices(self, data):
+        if self.stratified and data.domain.has_discrete_class:
             try:
                 splitter = skl.StratifiedKFold(
                     self.k, shuffle=True, random_state=self.random_state
                 )
-                splitter.get_n_splits(test_data.X, test_data.Y)
-                self.indices = list(splitter.split(test_data.X, test_data.Y))
+                splitter.get_n_splits(data.X, data.Y)
+                return list(splitter.split(data.X, data.Y))
             except ValueError:
                 self.warnings.append("Using non-stratified sampling.")
-                self.indices = None
-        if self.indices is None:
-            splitter = skl.KFold(
-                self.k, shuffle=True, random_state=self.random_state
-            )
-            splitter.get_n_splits(test_data)
-            self.indices = list(splitter.split(test_data))
 
+        splitter = skl.KFold(
+            self.k, shuffle=True, random_state=self.random_state)
+        splitter.get_n_splits(data)
+        return list(splitter.split(data))
 
-class CrossValidationFeature(Results):
+
+class CrossValidationFeature(Validation):
     """
     Cross validation with folds according to values of a feature.
 
-    .. attribute:: feature
-
-        The feature defining the folds.
-
+    Attributes:
+        feature (Orange.data.Variable): the feature defining the folds
     """
-    def __init__(self, data, learners, feature, store_data=False, store_models=False,
-                 preprocessor=None, callback=None, n_jobs=1):
+    def __init__(self, feature=None,
+                 store_data=False, store_models=False, warnings=None):
+        super().__init__(store_data=store_data, store_models=store_models)
         self.feature = feature
-        super().__init__(data, learners=learners, store_data=store_data,
-                         store_models=store_models, preprocessor=preprocessor,
-                         callback=callback, n_jobs=n_jobs)
 
-    def setup_indices(self, train_data, test_data):
-        data = Table(Domain([self.feature], None), test_data)
+    def get_indices(self, data):
+        data = Table(Domain([self.feature], None), data)
         values = data[:, self.feature].X
-        self.indices = []
+        indices = []
         for v in range(len(self.feature.values)):
             test_index = np.where(values == v)[0]
             train_index = np.where((values != v) & (~np.isnan(values)))[0]
-            if len(test_index) and len(train_index):
-                self.indices.append((train_index, test_index))
-        if not self.indices:
-            raise ValueError("No folds could be created from the given feature.")
+            if test_index.size and train_index.size:
+                indices.append((train_index, test_index))
+        if not indices:
+            raise ValueError(
+                f"'{self.feature.name}' does not have at least two distinct "
+                "values on the data")
+        return indices
 
 
-class LeaveOneOut(Results):
+class LeaveOneOut(Validation):
     """Leave-one-out testing"""
     score_by_folds = False
 
-    def __init__(self, data, learners, store_data=False, store_models=False,
-                 preprocessor=None, callback=None, n_jobs=1):
-        super().__init__(data, learners=learners, store_data=store_data,
-                         store_models=store_models, preprocessor=preprocessor,
-                         callback=callback, n_jobs=n_jobs)
-
-    def setup_indices(self, train_data, test_data):
+    def get_indices(self, data):
         splitter = skl.LeaveOneOut()
-        splitter.get_n_splits(test_data)
-        self.indices = list(splitter.split(test_data))
+        splitter.get_n_splits(data)
+        return list(splitter.split(data))
 
-    def prepare_arrays(self, test_data):
+    @staticmethod
+    def prepare_arrays(data, indices):
         # sped up version of super().prepare_arrays(data)
-        self.row_indices = np.arange(len(test_data))
-        self.folds = self.row_indices
-        self.actual = test_data.Y.flatten()
+        row_indices = np.arange(len(data))
+        return row_indices, row_indices, data.Y.flatten()
 
 
-class ShuffleSplit(Results):
-    def __init__(self, data, learners, n_resamples=10, train_size=None,
-                 test_size=0.1, stratified=True, random_state=0, store_data=False,
-                 store_models=False, preprocessor=None, callback=None, n_jobs=1):
+class ShuffleSplit(Validation):
+    """
+    Test by repeated random sampling
+
+    Attributes:
+        n_resamples (int): number of repetitions
+        test_size (float, int, None):
+            If float, should be between 0.0 and 1.0 and represent the proportion
+            of the dataset to include in the test split. If int, represents the
+            absolute number of test samples. If None, the value is set to the
+            complement of the train size. By default, the value is set to 0.1.
+            The default will change in version 0.21. It will remain 0.1 only
+            if ``train_size`` is unspecified, otherwise it will complement
+            the specified ``train_size``.
+            (from documentation of scipy.sklearn.StratifiedShuffleSplit)
+
+        train_size : float, int, or None, default is None
+            If float, should be between 0.0 and 1.0 and represent the
+            proportion of the dataset to include in the train split. If
+            int, represents the absolute number of train samples. If None,
+            the value is automatically set to the complement of the test size.
+            (from documentation of scipy.sklearn.StratifiedShuffleSplit)
+
+        stratified (bool):
+            flag deciding whether to perform stratified cross-validation.
+
+        random_state (int):
+            seed for random number generator (default: 0). If set to `None`,
+            a different seed is used each time
+
+    """
+    def __init__(self, n_resamples=10, train_size=None, test_size=0.1,
+                 stratified=True, random_state=0,
+                 store_data=False, store_models=False):
+        super().__init__(store_data=store_data, store_models=store_models)
         self.n_resamples = n_resamples
         self.train_size = train_size
         self.test_size = test_size
         self.stratified = stratified
         self.random_state = random_state
 
-        super().__init__(data, learners=learners, store_data=store_data,
-                         store_models=store_models, preprocessor=preprocessor,
-                         callback=callback, n_jobs=n_jobs)
-
-    def setup_indices(self, train_data, test_data):
-        if self.stratified and test_data.domain.has_discrete_class:
+    def get_indices(self, data):
+        if self.stratified and data.domain.has_discrete_class:
             splitter = skl.StratifiedShuffleSplit(
                 n_splits=self.n_resamples, train_size=self.train_size,
                 test_size=self.test_size, random_state=self.random_state
             )
-            splitter.get_n_splits(test_data.X, test_data.Y)
-            self.indices = list(splitter.split(test_data.X, test_data.Y))
+            splitter.get_n_splits(data.X, data.Y)
+            return list(splitter.split(data.X, data.Y))
 
-        else:
-            splitter = skl.ShuffleSplit(
-                n_splits=self.n_resamples, train_size=self.train_size,
-                test_size=self.test_size, random_state=self.random_state
-            )
-            splitter.get_n_splits(test_data)
-            self.indices = list(splitter.split(test_data))
+        splitter = skl.ShuffleSplit(
+            n_splits=self.n_resamples, train_size=self.train_size,
+            test_size=self.test_size, random_state=self.random_state
+        )
+        splitter.get_n_splits(data)
+        return list(splitter.split(data))
 
 
-class TestOnTestData(Results):
+class TestOnTestData(Validation):
     """
-    Test on a separate test dataset.
-    """
-    def __init__(self, train_data, test_data, learners, store_data=False,
-                 store_models=False, preprocessor=None, callback=None, n_jobs=1):
-        super().__init__(test_data, train_data=train_data, learners=learners,
-                         store_data=store_data,
-                         store_models=store_models, preprocessor=preprocessor,
-                         callback=callback, n_jobs=n_jobs)
-
-    def setup_indices(self, train_data, test_data):
-        self.indices = ((Ellipsis, Ellipsis),)
-
-    def prepare_arrays(self, test_data):
-        self.row_indices = np.arange(len(test_data))
-        self.folds = (Ellipsis, )
-        self.actual = test_data.Y.ravel()
-
+    Test on separately provided test data
 
-class TestOnTrainingData(TestOnTestData):
-    """
-    Trains and test on the same data
+    Note that the class has a different signature for `__call__`.
     """
+    # get_indices is not needed in this class, pylint: disable=abstract-method
+
+    def __new__(cls, data=None, test_data=None, learners=None,
+                preprocessor=None, **kwargs):
+        if "train_data" in kwargs:
+            if data is None:
+                data = kwargs.pop("train_data")
+            else:
+                raise ValueError(
+                    "argument 'data' is given twice (once as 'train_data')")
+        return super().__new__(
+            cls,
+            data=data, learners=learners, preprocessor=preprocessor,
+            test_data=test_data, **kwargs)
+
+    def __call__(self, data, test_data, learners, preprocessor=None,
+                 *, callback=None):
+        """
+        Args:
+            data (Orange.data.Table): training data
+            test_data (Orange.data.Table): test_data
+            learners (list of Orange.Learner): a list of learning algorithms
+            preprocessor (Orange.preprocess.Preprocess): preprocessor applied
+                on training data
+            callback (Callable): a function called to notify about the progress
 
-    def __init__(self, data, learners, store_data=False, store_models=False,
-                 preprocessor=None, callback=None, n_jobs=1):
+        Returns:
+            results (Result): results of testing
+        """
+        if preprocessor is None:
+            preprocessor = _identity
+        if callback is None:
+            callback = _identity
+
+        train_data = preprocessor(data)
+        part_results = []
+        for (learner_i, learner) in enumerate(learners):
+            part_results.append(
+                _mp_worker(0, train_data, test_data, learner_i, learner,
+                           self.store_models))
+            callback((learner_i + 1) / len(learners))
+        callback(1)
+
+        results = Results(
+            data=test_data if self.store_data else None,
+            domain=test_data.domain,
+            nrows=len(test_data), learners=learners,
+            row_indices=np.arange(len(test_data)),
+            folds=(Ellipsis, ),
+            actual=test_data.Y.ravel(),
+            score_by_folds=self.score_by_folds)
+        if self.store_models:
+            results.models = np.tile(None, (1, len(learners)))
+        self._collect_part_results(results, part_results)
+        return results
 
-        if preprocessor is not None:
-            data = preprocessor(data)
 
-        super().__init__(train_data=data, test_data=data, learners=learners,
-                         store_data=store_data, store_models=store_models,
-                         preprocessor=None, callback=callback, n_jobs=n_jobs)
-        self.preprocessor = preprocessor
+class TestOnTrainingData(TestOnTestData):
+    """Test on training data"""
+    # get_indices is not needed in this class, pylint: disable=abstract-method
+    # signature is such as on the base class, pylint: disable=signature-differs
+    def __new__(cls, data=None, learners=None, preprocessor=None, **kwargs):
+        return super().__new__(
+            cls,
+            data, test_data=data, learners=learners, preprocessor=preprocessor,
+            **kwargs)
+
+    def __call__(self, data, learners, preprocessor=None, *, callback=None,
+                 **kwargs):
+        kwargs.setdefault("test_data", data)
+        # if kwargs contains anything besides test_data, this will be detected
+        # (and complained about) by super().__call__
+        return super().__call__(
+            data=data, learners=learners, preprocessor=preprocessor,
+            callback=callback, **kwargs)
 
 
 def sample(table, n=0.7, stratified=False, replace=False,
@@ -601,7 +766,7 @@ def sample(table, n=0.7, stratified=False, replace=False,
         Pseudo-random number generator state used for random sampling.
     """
 
-    if type(n) == float:
+    if isinstance(n, float):
         n = int(n * len(table))
 
     if replace:
@@ -609,11 +774,11 @@ def sample(table, n=0.7, stratified=False, replace=False,
             rgen = np.random
         else:
             rgen = np.random.mtrand.RandomState(random_state)
-        sample = rgen.randint(0, len(table), n)
+        a_sample = rgen.randint(0, len(table), n)
         o = np.ones(len(table))
-        o[sample] = 0
+        o[a_sample] = 0
         others = np.nonzero(o)[0]
-        return table[sample], table[others]
+        return table[a_sample], table[others]
 
     n = len(table) - n
     if stratified and table.domain.has_discrete_class:
diff --git a/Orange/tests/test_evaluation_clustering.py b/Orange/tests/test_evaluation_clustering.py
--- a/Orange/tests/test_evaluation_clustering.py
+++ b/Orange/tests/test_evaluation_clustering.py
@@ -8,11 +8,12 @@
 import Orange
 from Orange.evaluation.clustering import Silhouette, \
     AdjustedMutualInfoScore, ClusteringEvaluation, ClusteringResults
-from Orange.clustering.kmeans import KMeans
+from Orange.clustering.kmeans import KMeans, KMeansModel
 
 
-class TestClusteringEvaluation(unittest.TestCase):
-    def test_init(self):
+class TestClusteringResults(unittest.TestCase):
+    @staticmethod
+    def test_init():
         data = Orange.data.Table(np.arange(100).reshape((100, 1)))
         res = ClusteringResults(data=data, nmethods=2, nrows=100)
         res.actual[:50] = 0
@@ -21,13 +22,26 @@ def test_init(self):
         expected = [1.0, 1.0]
         np.testing.assert_almost_equal(AdjustedMutualInfoScore(res), expected)
 
+
+class TestClusteringEvaluation(unittest.TestCase):
+    def test_init(self):
+        res = ClusteringEvaluation(k=42)
+        self.assertEqual(res.k, 42)
+
     def test_kmeans(self):
         table = Orange.data.Table('iris')
-        cr = ClusteringEvaluation(table, learners=[KMeans(n_clusters=2),
-                                                   KMeans(n_clusters=3),
-                                                   KMeans(n_clusters=5)], k=3)
+        cr = ClusteringEvaluation(k=3)(table, learners=[KMeans(n_clusters=2),
+                                                        KMeans(n_clusters=3),
+                                                        KMeans(n_clusters=5)])
         expected = [0.68081362, 0.55259194, 0.48851755]
         np.testing.assert_almost_equal(Silhouette(cr), expected, decimal=2)
         expected = [0.51936073, 0.74837231, 0.59178896]
         np.testing.assert_almost_equal(AdjustedMutualInfoScore(cr),
                                        expected, decimal=2)
+        self.assertIsNone(cr.models)
+
+        cr = ClusteringEvaluation(k=3, store_models=True)(
+            table, learners=[KMeans(n_clusters=2)])
+        self.assertEqual(cr.models.shape, (3, 1))
+        self.assertTrue(all(isinstance(m, KMeansModel)
+                            for m in cr.models.flatten()))
diff --git a/Orange/tests/test_evaluation_scoring.py b/Orange/tests/test_evaluation_scoring.py
--- a/Orange/tests/test_evaluation_scoring.py
+++ b/Orange/tests/test_evaluation_scoring.py
@@ -56,7 +56,7 @@ def setUpClass(cls):
 
     def test_precision_iris(self):
         learner = LogisticRegressionLearner(preprocessors=[])
-        res = TestOnTrainingData(self.iris, [learner])
+        res = TestOnTrainingData()(self.iris, [learner])
         self.assertAlmostEqual(self.score(res, average='weighted')[0],
                                0.96189, 5)
         self.assertAlmostEqual(self.score(res, target=1)[0], 0.97826, 5)
@@ -115,7 +115,7 @@ def setUpClass(cls):
 
     def test_recall_iris(self):
         learner = LogisticRegressionLearner(preprocessors=[])
-        res = TestOnTrainingData(self.iris, [learner])
+        res = TestOnTrainingData()(self.iris, [learner])
         self.assertAlmostEqual(self.score(res, average='weighted')[0], 0.96, 5)
         self.assertAlmostEqual(self.score(res, target=1)[0], 0.9, 5)
         self.assertAlmostEqual(self.score(res, target=1, average=None)[0],
@@ -173,7 +173,7 @@ def setUpClass(cls):
 
     def test_recall_iris(self):
         learner = LogisticRegressionLearner(preprocessors=[])
-        res = TestOnTrainingData(self.iris, [learner])
+        res = TestOnTrainingData()(self.iris, [learner])
         self.assertAlmostEqual(self.score(res, average='weighted')[0],
                                0.959935, 5)
         self.assertAlmostEqual(self.score(res, target=1)[0], 0.9375, 5)
@@ -257,11 +257,11 @@ def test_bayes(self):
         t = Discretize(
             method=discretize.EqualWidth(n=3))(t)
         nb = NaiveBayesLearner()
-        res = TestOnTrainingData(t, [nb])
+        res = TestOnTrainingData()(t, [nb])
         np.testing.assert_almost_equal(CA(res), [1])
 
         t.Y[-20:] = 1 - t.Y[-20:]
-        res = TestOnTrainingData(t, [nb])
+        res = TestOnTrainingData()(t, [nb])
         self.assertGreaterEqual(CA(res)[0], 0.75)
         self.assertLess(CA(res)[0], 1)
 
@@ -273,19 +273,19 @@ def setUpClass(cls):
 
     def test_tree(self):
         tree = SklTreeLearner()
-        res = CrossValidation(self.iris, [tree], k=2)
+        res = CrossValidation(k=2)(self.iris, [tree])
         self.assertGreater(AUC(res)[0], 0.8)
         self.assertLess(AUC(res)[0], 1.)
 
     def test_constant_prob(self):
         maj = MajorityLearner()
-        res = TestOnTrainingData(self.iris, [maj])
+        res = TestOnTrainingData()(self.iris, [maj])
         self.assertEqual(AUC(res)[0], 0.5)
 
     def test_multiclass_auc_multi_learners(self):
         learners = [LogisticRegressionLearner(),
                     MajorityLearner()]
-        res = CrossValidation(self.iris, learners, k=10)
+        res = CrossValidation(k=10)(self.iris, learners)
         self.assertGreater(AUC(res)[0], 0.6)
         self.assertLess(AUC(res)[1], 0.6)
         self.assertGreater(AUC(res)[1], 0.4)
@@ -295,11 +295,11 @@ def test_auc_on_multiclass_data_returns_1d_array(self):
         lenses = Table(test_filename('datasets/lenses.tab'))[:100]
         majority = MajorityLearner()
 
-        results = TestOnTrainingData(lenses, [majority])
+        results = TestOnTrainingData()(lenses, [majority])
         auc = AUC(results)
         self.assertEqual(auc.ndim, 1)
 
-        results = TestOnTrainingData(titanic, [majority])
+        results = TestOnTrainingData()(titanic, [majority])
         auc = AUC(results)
         self.assertEqual(auc.ndim, 1)
 
@@ -343,7 +343,7 @@ class TestLogLoss(unittest.TestCase):
     def test_log_loss(self):
         data = Table('iris')
         majority = MajorityLearner()
-        results = TestOnTrainingData(data, [majority])
+        results = TestOnTrainingData()(data, [majority])
         ll = LogLoss(results)
         self.assertAlmostEqual(ll[0], - np.log(1 / 3))
 
@@ -355,7 +355,7 @@ def _log_loss(self, act, prob):
     def test_log_loss_calc(self):
         data = Table('titanic')
         learner = LogisticRegressionLearner()
-        results = TestOnTrainingData(data, [learner])
+        results = TestOnTrainingData()(data, [learner])
 
         actual = np.copy(results.actual)
         actual = actual.reshape(actual.shape[0], 1)
diff --git a/Orange/tests/test_evaluation_testing.py b/Orange/tests/test_evaluation_testing.py
--- a/Orange/tests/test_evaluation_testing.py
+++ b/Orange/tests/test_evaluation_testing.py
@@ -2,9 +2,12 @@
 # pylint: disable=missing-docstring
 
 import unittest
+from unittest.mock import Mock, patch
+
 import numpy as np
 
 from Orange.classification import NaiveBayesLearner, MajorityLearner
+from Orange.evaluation.testing import Validation
 from Orange.regression import LinearRegressionLearner, MeanLearner
 from Orange.data import Table, Domain, DiscreteVariable
 from Orange.evaluation import (Results, CrossValidation, LeaveOneOut, TestOnTrainingData,
@@ -25,7 +28,7 @@ def random_data(nrows, ncols):
 
 class TestingTestCase(unittest.TestCase):
     def test_no_data(self):
-        self.assertRaises(TypeError, CrossValidation,
+        self.assertRaises(ValueError, CrossValidation,
                           learners=[NaiveBayesLearner()])
 
 
@@ -60,7 +63,7 @@ def fails(_):
 
         major_call = 0
         fail_calls = 0
-        res = method(random_data(50, 4), [major, fails, major])
+        res = method()(random_data(50, 4), [major, fails, major])
         self.assertFalse(res.failed[0])
         self.assertIsInstance(res.failed[1], Exception)
         self.assertFalse(res.failed[2])
@@ -71,8 +74,9 @@ def run_test_callback(self, method, expected_progresses):
         def record_progress(p):
             progress.append(p)
         progress = []
-        method(self.random_table, [MajorityLearner(), MajorityLearner()],
-               callback=record_progress)
+        method()(
+            self.random_table, [MajorityLearner(), MajorityLearner()],
+            callback=record_progress)
         np.testing.assert_almost_equal(np.array(progress), expected_progresses)
 
     def run_test_preprocessor(self, method, expected_sizes):
@@ -80,8 +84,8 @@ def preprocessor(data):
             data_sizes.append(len(data))
             return data
         data_sizes = []
-        method(Table('iris'), [MajorityLearner(), MajorityLearner()],
-               preprocessor=preprocessor)
+        method()(Table('iris'), [MajorityLearner(), MajorityLearner()],
+                 preprocessor=preprocessor)
         self.assertEqual(data_sizes, expected_sizes)
 
     def check_folds(self, result, folds_count, rows):
@@ -97,10 +101,62 @@ def check_models(self, result, learners, folds):
             for model, learner in zip(models, learners):
                 self.assertIsInstance(model, learner.__returns__)
 
-    def _callback_values(self, iterations):
+    @staticmethod
+    def _callback_values(iterations):
         return np.hstack((np.linspace(0., .99, iterations + 1)[1:], [1]))
 
 
+class TestValidation(unittest.TestCase):
+    def setUp(self):
+        self.data = Table('iris')
+
+    def test_invalid_argument_combination(self):
+        self.assertRaises(ValueError, Validation, self.data)
+        self.assertRaises(ValueError, Validation, None, [MajorityLearner()])
+        self.assertRaises(ValueError, Validation, preprocessor=lambda x: x)
+        self.assertRaises(ValueError, Validation, callback=lambda x: x)
+
+    @patch("Orange.evaluation.testing.Validation.__call__")
+    def test_warn_deprecations(self, _):
+        self.assertWarns(
+            DeprecationWarning,
+            Validation, self.data, [MajorityLearner()])
+
+        self.assertWarns(DeprecationWarning, Validation().fit)
+
+    @patch("Orange.evaluation.testing.Validation.__call__")
+    def test_obsolete_call_constructor(self, validation_call):
+
+        class MockValidation(Validation):
+            args = kwargs = None
+
+            def __init__(self, *args, **kwargs):
+                super().__init__()
+                MockValidation.args = args
+                MockValidation.kwargs = kwargs
+
+            def get_indices(self, data):
+                pass
+
+        data = self.data
+        learners = [MajorityLearner(), MajorityLearner()]
+        kwargs = dict(foo=42, store_data=43, store_models=44, callback=45, n_jobs=46)
+        self.assertWarns(
+            DeprecationWarning,
+            MockValidation, data, learners=learners,
+            **kwargs)
+        self.assertEqual(MockValidation.args, ())
+        kwargs.pop("n_jobs")  # do not pass n_jobs and callback from __new__ to __init__
+        kwargs.pop("callback")
+        self.assertEqual(MockValidation.kwargs, kwargs)
+
+        cargs, ckwargs = validation_call.call_args
+        self.assertEqual(len(cargs), 1)
+        self.assertIs(cargs[0], data)
+        self.assertIs(ckwargs["learners"], learners)
+        self.assertEqual(ckwargs["callback"], 45)
+
+
 class TestCrossValidation(TestSampling):
     @classmethod
     def setUpClass(cls):
@@ -110,10 +166,16 @@ def setUpClass(cls):
         cls.ncols = 5
         cls.random_table = random_data(cls.nrows, cls.ncols)
 
+    def test_init(self):
+        res = CrossValidation(k=42, stratified=False, random_state=43)
+        self.assertEqual(res.k, 42)
+        self.assertFalse(res.stratified)
+        self.assertEqual(res.random_state, 43)
+
     def test_results(self):
-        nrows, ncols = self.random_table.X.shape
-        res = CrossValidation(self.random_table, [NaiveBayesLearner()], k=10,
-                              stratified=False)
+        nrows, _ = self.random_table.X.shape
+        res = CrossValidation(k=10, stratified=False)(
+            self.random_table, [NaiveBayesLearner()])
         y = self.random_table.Y
         np.testing.assert_equal(res.actual, y[res.row_indices].reshape(nrows))
         np.testing.assert_equal(res.predicted[0],
@@ -123,17 +185,18 @@ def test_results(self):
         self.check_folds(res, 10, nrows)
 
     def test_continuous(self):
-        res = CrossValidation(self.housing, [LinearRegressionLearner()], k=3, n_jobs=1)
+        res = CrossValidation(k=3)(
+            self.housing, [LinearRegressionLearner()])
         self.assertLess(RMSE(res), 5)
 
     def test_folds(self):
-        res = CrossValidation(self.random_table, [NaiveBayesLearner()], k=5)
+        res = CrossValidation(k=5)(self.random_table, [NaiveBayesLearner()])
         self.check_folds(res, 5, self.nrows)
 
     def test_call_5(self):
-        nrows, ncols = self.random_table.X.shape
-        res = CrossValidation(self.random_table, [NaiveBayesLearner()], k=5,
-                              stratified=False)
+        nrows, _ = self.random_table.X.shape
+        res = CrossValidation(k=5, stratified=False)(
+            self.random_table, [NaiveBayesLearner()])
         y = self.random_table.Y
         np.testing.assert_equal(res.actual, y[res.row_indices].reshape(nrows))
         np.testing.assert_equal(res.predicted[0],
@@ -144,25 +207,26 @@ def test_call_5(self):
 
     def test_store_data(self):
         learners = [NaiveBayesLearner()]
-        res = CrossValidation(self.random_table, learners, store_data=False)
+        res = CrossValidation(store_data=False)(self.random_table, learners)
         self.assertIsNone(res.data)
 
-        res = CrossValidation(self.random_table, learners, store_data=True)
+        res = CrossValidation(store_data=True)(self.random_table, learners)
         self.assertIs(res.data, self.random_table)
 
     def test_store_models(self):
         learners = [NaiveBayesLearner(), MajorityLearner()]
 
-        res = CrossValidation(self.random_table, learners, k=5, store_models=False)
+        res = CrossValidation(store_models=False, k=5)(
+            self.random_table, learners)
         self.assertIsNone(res.models)
 
-        res = CrossValidation(self.random_table, learners, k=5, store_models=True)
+        res = CrossValidation(k=5, store_models=True)(self.random_table, learners)
         self.assertEqual(len(res.models), 5)
         self.check_models(res, learners, 5)
 
     def test_split_by_model(self):
         learners = [NaiveBayesLearner(), MajorityLearner()]
-        res = CrossValidation(self.random_table, learners, k=5, store_models=True)
+        res = CrossValidation(k=5, store_models=True)(self.random_table, learners)
 
         for i, result in enumerate(res.split_by_model()):
             self.assertIsInstance(result, Results)
@@ -176,7 +240,7 @@ def test_split_by_model(self):
     def test_10_fold_probs(self):
         learners = [MajorityLearner(), MajorityLearner()]
 
-        results = CrossValidation(self.iris[30:130], learners, k=10)
+        results = CrossValidation(k=10)(self.iris[30:130], learners)
 
         self.assertEqual(results.predicted.shape, (2, len(self.iris[30:130])))
         np.testing.assert_equal(results.predicted, np.ones((2, 100)))
@@ -184,21 +248,24 @@ def test_10_fold_probs(self):
         self.assertTrue((probs[:, :, 0] < probs[:, :, 2]).all())
         self.assertTrue((probs[:, :, 2] < probs[:, :, 1]).all())
 
-    def test_miss_majority(self):
+    @staticmethod
+    def test_miss_majority():
         x = np.zeros((50, 3))
         y = x[:, -1]
         x[-4:] = np.ones((4, 3))
         data = Table(x, y)
-        res = CrossValidation(data, [MajorityLearner()], k=3)
+        cv = CrossValidation(k=3)
+        res = cv(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0][:49], 0)
 
         x[-4:] = np.zeros((4, 3))
-        res = CrossValidation(data, [MajorityLearner()], k=3)
+        res = cv(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0][:49], 0)
 
     def test_too_many_folds(self):
         w = []
-        res = CrossValidation(self.iris, [MajorityLearner()], k=len(self.iris)/2, warnings=w)
+        CrossValidation(k=len(self.iris) // 2, warnings=w)(
+            self.iris, [MajorityLearner()])
         self.assertGreater(len(w), 0)
 
     def test_failed(self):
@@ -213,7 +280,7 @@ def test_preprocessor(self):
     def test_augmented_data_classification(self):
         data = Table("iris")
         n_classes = len(data.domain.class_var.values)
-        res = CrossValidation(data, [NaiveBayesLearner()], store_data=True)
+        res = CrossValidation(store_data=True)(data, [NaiveBayesLearner()])
         table = res.get_augmented_data(['Naive Bayes'])
 
         self.assertEqual(len(table), len(data))
@@ -225,7 +292,7 @@ def test_augmented_data_classification(self):
         self.assertEqual(
             table.domain.metas[len(data.domain.metas)].values, data.domain.class_var.values)
 
-        res = CrossValidation(data, [NaiveBayesLearner(), MajorityLearner()], store_data=True)
+        res = CrossValidation(store_data=True)(data, [NaiveBayesLearner(), MajorityLearner()])
         table = res.get_augmented_data(['Naive Bayes', 'Majority'])
 
         self.assertEqual(len(table), len(data))
@@ -240,7 +307,7 @@ def test_augmented_data_classification(self):
 
     def test_augmented_data_regression(self):
         data = Table("housing")
-        res = CrossValidation(data, [LinearRegressionLearner(), ], store_data=True)
+        res = CrossValidation(store_data=True)(data, [LinearRegressionLearner()])
         table = res.get_augmented_data(['Linear Regression'])
 
         self.assertEqual(len(table), len(data))
@@ -249,7 +316,7 @@ def test_augmented_data_regression(self):
         # +1 for class, +1 for fold
         self.assertEqual(len(table.domain.metas), len(data.domain.metas) + 1 + 1)
 
-        res = CrossValidation(data, [LinearRegressionLearner(), MeanLearner()], store_data=True)
+        res = CrossValidation(store_data=True)(data, [LinearRegressionLearner(), MeanLearner()])
         table = res.get_augmented_data(['Linear Regression', 'Mean Learner'])
 
         self.assertEqual(len(table), len(data))
@@ -261,7 +328,8 @@ def test_augmented_data_regression(self):
 
 class TestCrossValidationFeature(TestSampling):
 
-    def add_meta_fold(self, data, f):
+    @staticmethod
+    def add_meta_fold(data, f):
         fat = DiscreteVariable(name="fold", values=[str(a) for a in range(f)])
         domain = Domain(data.domain.attributes, data.domain.class_var, metas=[fat])
         ndata = Table(domain, data)
@@ -270,10 +338,15 @@ def add_meta_fold(self, data, f):
         ndata[:, fat] = vals
         return ndata
 
+    def test_init(self):
+        var = DiscreteVariable(name="fold", values="abc")
+        res = CrossValidationFeature(feature=var)
+        self.assertIs(res.feature, var)
+
     def test_call(self):
         t = self.random_table
         t = self.add_meta_fold(t, 3)
-        res = CrossValidationFeature(t, [NaiveBayesLearner()], feature=t.domain.metas[0])
+        res = CrossValidationFeature(feature=t.domain.metas[0])(t, [NaiveBayesLearner()])
         y = t.Y
         np.testing.assert_equal(res.actual, y[res.row_indices].reshape(len(t)))
         np.testing.assert_equal(res.predicted[0],
@@ -286,15 +359,27 @@ def test_unknown(self):
         t = self.add_meta_fold(t, 3)
         fat = t.domain.metas[0]
         t[0][fat] = float("nan")
-        res = CrossValidationFeature(t, [NaiveBayesLearner()], feature=fat)
+        res = CrossValidationFeature(feature=fat)(t, [NaiveBayesLearner()])
         self.assertNotIn(0, res.row_indices)
 
+    def test_bad_feature(self):
+        feat = DiscreteVariable(name="fold", values="abc")
+        domain = Domain([DiscreteVariable("x", values="ab")],
+                        DiscreteVariable("y", values="cd"),
+                        metas=[feat])
+        t = Table.from_numpy(
+            domain,
+            np.zeros((10, 1)), np.ones((10, 1)), np.full((10, 1), np.nan))
+        self.assertRaises(
+            ValueError,
+            CrossValidationFeature(feature=feat), t, [NaiveBayesLearner()])
+
 
 class TestLeaveOneOut(TestSampling):
     def test_results(self):
         nrows = self.nrows
         t = self.random_table
-        res = LeaveOneOut(t, [NaiveBayesLearner()])
+        res = LeaveOneOut()(t, [NaiveBayesLearner()])
         y = t.Y
         np.testing.assert_equal(res.actual, y[res.row_indices].reshape(nrows))
         np.testing.assert_equal(res.predicted[0],
@@ -306,7 +391,7 @@ def test_results(self):
     def test_call(self):
         nrows = self.nrows
         t = self.random_table
-        res = LeaveOneOut(t, [NaiveBayesLearner()])
+        res = LeaveOneOut()(t, [NaiveBayesLearner()])
         y = t.Y
         np.testing.assert_equal(res.actual, y[res.row_indices].reshape(nrows))
         np.testing.assert_equal(res.predicted[0],
@@ -318,27 +403,27 @@ def test_store_data(self):
         t = self.random_table
         learners = [NaiveBayesLearner()]
 
-        res = LeaveOneOut(t, learners, store_data=False)
+        res = LeaveOneOut(store_data=False)(t, learners)
         self.assertIsNone(res.data)
 
-        res = LeaveOneOut(t, learners, store_data=True)
+        res = LeaveOneOut(store_data=True)(t, learners)
         self.assertIs(res.data, t)
 
     def test_store_models(self):
         t = self.random_table
         learners = [NaiveBayesLearner(), MajorityLearner()]
 
-        res = LeaveOneOut(t, learners)
+        res = LeaveOneOut()(t, learners)
         self.assertIsNone(res.models)
 
-        res = LeaveOneOut(t, learners, store_models=True)
+        res = LeaveOneOut(store_models=True)(t, learners)
         self.check_models(res, learners, self.nrows)
 
     def test_probs(self):
         data = Table('iris')[30:130]
         learners = [MajorityLearner(), MajorityLearner()]
 
-        results = LeaveOneOut(data, learners)
+        results = LeaveOneOut()(data, learners)
 
         self.assertEqual(results.predicted.shape, (2, len(data)))
         np.testing.assert_equal(results.predicted, np.ones((2, 100)))
@@ -346,21 +431,22 @@ def test_probs(self):
         self.assertTrue((probs[:, :, 0] < probs[:, :, 2]).all())
         self.assertTrue((probs[:, :, 2] < probs[:, :, 1]).all())
 
-    def test_miss_majority(self):
+    @staticmethod
+    def test_miss_majority():
         x = np.zeros((50, 3))
         y = x[:, -1]
         x[49] = 1
         data = Table(x, y)
-        res = LeaveOneOut(data, [MajorityLearner()])
+        res = LeaveOneOut()(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0][:49], 0)
 
         x[49] = 0
-        res = LeaveOneOut(data, [MajorityLearner()])
+        res = LeaveOneOut()(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0][:49], 0)
 
         x[25:] = 1
         data = Table(x, y)
-        res = LeaveOneOut(data, [MajorityLearner()])
+        res = LeaveOneOut()(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0],
                                 1 - data.Y[res.row_indices].flatten())
 
@@ -375,10 +461,14 @@ def test_preprocessor(self):
 
 
 class TestTestOnTrainingData(TestSampling):
+    @staticmethod
+    def _callback_values(iterations):
+        return np.hstack(((np.arange(iterations) + 1) / iterations, [1]))
+
     def test_results(self):
-        nrows, ncols = self.random_table.X.shape
+        nrows, _ = self.random_table.X.shape
         t = self.random_table
-        res = TestOnTrainingData(t, [NaiveBayesLearner()])
+        res = TestOnTrainingData()(t, [NaiveBayesLearner()])
         y = t.Y
         np.testing.assert_equal(res.actual, y[res.row_indices].reshape(nrows))
         np.testing.assert_equal(res.predicted[0],
@@ -390,26 +480,26 @@ def test_results(self):
     def test_store_data(self):
         t = self.random_table
         learners = [NaiveBayesLearner()]
-        res = TestOnTrainingData(t, learners)
+        res = TestOnTrainingData()(t, learners)
         self.assertIsNone(res.data)
-        res = TestOnTrainingData(t, learners, store_data=True)
+        res = TestOnTrainingData(store_data=True)(t, learners)
         self.assertIs(res.data, t)
 
     def test_store_models(self):
         t = self.random_table
         learners = [NaiveBayesLearner(), MajorityLearner()]
 
-        res = TestOnTrainingData(t, learners)
+        res = TestOnTrainingData()(t, learners)
         self.assertIsNone(res.models)
 
-        res = TestOnTrainingData(t, learners, store_models=True)
+        res = TestOnTrainingData(store_models=True)(t, learners)
         self.check_models(res, learners, 1)
 
     def test_probs(self):
         data = self.iris[30:130]
         learners = [MajorityLearner(), MajorityLearner()]
 
-        results = TestOnTrainingData(data, learners)
+        results = TestOnTrainingData()(data, learners)
 
         self.assertEqual(results.predicted.shape, (2, len(data)))
         np.testing.assert_equal(results.predicted, np.ones((2, 100)))
@@ -417,21 +507,22 @@ def test_probs(self):
         self.assertTrue((probs[:, :, 0] < probs[:, :, 2]).all())
         self.assertTrue((probs[:, :, 2] < probs[:, :, 1]).all())
 
-    def test_miss_majority(self):
+    @staticmethod
+    def test_miss_majority():
         x = np.zeros((50, 3))
         y = x[:, -1]
         x[49] = 1
         data = Table(x, y)
-        res = TestOnTrainingData(data, [MajorityLearner()])
+        res = TestOnTrainingData()(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0][:49], 0)
 
         x[49] = 0
-        res = TestOnTrainingData(data, [MajorityLearner()])
+        res = TestOnTrainingData()(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0][:49], 0)
 
         x[25:] = 1
         data = Table(x, y)
-        res = TestOnTrainingData(data, [MajorityLearner()])
+        res = TestOnTrainingData()(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0], res.predicted[0][0])
 
     def test_failed(self):
@@ -445,10 +536,14 @@ def test_preprocessor(self):
 
 
 class TestTestOnTestData(TestSampling):
+    @staticmethod
+    def _callback_values(iterations):
+        return np.hstack(((np.arange(iterations) + 1) / iterations, [1]))
+
     def test_results(self):
-        nrows, ncols = self.random_table.X.shape
+        nrows, _ = self.random_table.X.shape
         t = self.random_table
-        res = TestOnTestData(t, t, [NaiveBayesLearner()])
+        res = TestOnTestData()(t, t, [NaiveBayesLearner()])
         y = t.Y
         np.testing.assert_equal(res.actual, y[res.row_indices].reshape(nrows))
         np.testing.assert_equal(res.predicted[0],
@@ -460,7 +555,7 @@ def test_results(self):
     def test_probs(self):
         data = self.iris[30:130]
         learners = [MajorityLearner(), MajorityLearner()]
-        results = TestOnTestData(data, data, learners)
+        results = TestOnTestData()(data, data, learners)
 
         self.assertEqual(results.predicted.shape, (2, len(data)))
         np.testing.assert_equal(results.predicted, np.ones((2, 100)))
@@ -470,7 +565,7 @@ def test_probs(self):
 
         train = self.iris[50:120]
         test = self.iris[:50]
-        results = TestOnTestData(train, test, learners)
+        results = TestOnTestData()(train, test, learners)
         self.assertEqual(results.predicted.shape, (2, len(test)))
         np.testing.assert_equal(results.predicted, np.ones((2, 50)))
         probs = results.probabilities
@@ -482,10 +577,10 @@ def test_store_data(self):
         test = data[int(self.nrows*.75):]
         learners = [MajorityLearner()]
 
-        res = TestOnTestData(train, test, learners)
+        res = TestOnTestData()(train, test, learners)
         self.assertIsNone(res.data)
 
-        res = TestOnTestData(train, test, learners, store_data=True)
+        res = TestOnTestData(store_data=True)(train, test, learners)
         self.assertIs(res.data, test)
 
     def test_store_models(self):
@@ -494,28 +589,29 @@ def test_store_models(self):
         test = data[int(self.nrows*.75):]
         learners = [NaiveBayesLearner(), MajorityLearner()]
 
-        res = TestOnTestData(train, test, learners)
+        res = TestOnTestData()(train, test, learners)
         self.assertIsNone(res.models)
 
-        res = TestOnTestData(train, test, learners, store_models=True)
+        res = TestOnTestData(store_models=True)(train, test, learners)
         self.check_models(res, learners, 1)
 
-    def test_miss_majority(self):
+    @staticmethod
+    def test_miss_majority():
         x = np.zeros((50, 3))
         y = x[:, -1]
         x[49] = 1
         data = Table(x, y)
-        res = TestOnTrainingData(data, [MajorityLearner()])
+        res = TestOnTrainingData()(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0][:49], 0)
 
         x[49] = 0
-        res = TestOnTrainingData(data, [MajorityLearner()])
+        res = TestOnTrainingData()(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0][:49], 0)
 
         x[25:] = 1
         y = x[:, -1]
         data = Table(x, y)
-        res = TestOnTrainingData(data, [MajorityLearner()])
+        res = TestOnTrainingData()(data, [MajorityLearner()])
         np.testing.assert_equal(res.predicted[0], res.predicted[0][0])
 
     def run_test_failed(self, method, succ_calls):
@@ -534,7 +630,7 @@ def fails(_):
         major_call = 0
         fail_calls = 0
         data = random_data(50, 4)
-        res = TestOnTestData(data, data, [major, fails, major])
+        res = TestOnTestData()(data, data, [major, fails, major])
         self.assertFalse(res.failed[0])
         self.assertIsInstance(res.failed[1], Exception)
         self.assertFalse(res.failed[2])
@@ -547,8 +643,8 @@ def record_progress(p):
 
         progress = []
         data = random_data(50, 4)
-        TestOnTestData(data, data, [MajorityLearner(), MajorityLearner()],
-                       callback=record_progress)
+        TestOnTestData()(data, data, [MajorityLearner(), MajorityLearner()],
+                         callback=record_progress)
         np.testing.assert_almost_equal(progress, self._callback_values(2))
 
     def test_preprocessor(self):
@@ -558,11 +654,31 @@ def preprocessor(data):
 
         data_sizes = []
         data = random_data(50, 5)
-        TestOnTestData(data[:30], data[-20:],
-                       [MajorityLearner(), MajorityLearner()],
-                       preprocessor=preprocessor)
+        TestOnTestData()(data[:30], data[-20:],
+                         [MajorityLearner(), MajorityLearner()],
+                         preprocessor=preprocessor)
         self.assertEqual(data_sizes, [30])
 
+    @patch("Orange.evaluation.testing.Validation.__new__")
+    def test_train_data_argument(self, validation_new):
+        data = Mock()
+        test_data = Mock()
+        TestOnTestData(data, test_data)
+        args = validation_new.call_args[1]
+        self.assertIs(args["data"], data)
+        self.assertIs(args["test_data"], test_data)
+        validation_new.reset_mock()
+
+        TestOnTestData(test_data=test_data, train_data=data)
+        args = validation_new.call_args[1]
+        self.assertIs(args["data"], data)
+        self.assertIs(args["test_data"], test_data)
+        validation_new.reset_mock()
+
+        self.assertRaises(
+            ValueError,
+            TestOnTestData, data, train_data=data, test_data=test_data)
+
 
 class TestTrainTestSplit(unittest.TestCase):
     def test_fixed_training_size(self):
@@ -588,38 +704,183 @@ def test_fixed_training_size(self):
 
 
 class TestShuffleSplit(TestSampling):
+    def test_init(self):
+        res = ShuffleSplit(n_resamples=1, train_size=0.1, test_size=0.2,
+                           stratified=False, random_state=42)
+        self.assertEqual(res.n_resamples, 1)
+        self.assertEqual(res.train_size, 0.1)
+        self.assertEqual(res.test_size, 0.2)
+        self.assertFalse(res.stratified)
+        self.assertEqual(res.random_state, 42)
+
     def test_results(self):
         data = self.random_table
         train_size, n_resamples = 0.6, 10
-        res = ShuffleSplit(data, [NaiveBayesLearner()], train_size=train_size,
-                           test_size=1 - train_size, n_resamples=n_resamples)
+        res = ShuffleSplit(
+            train_size=train_size, test_size=1 - train_size,
+            n_resamples=n_resamples)(data, [NaiveBayesLearner()])
         self.assertEqual(len(res.predicted[0]),
                          n_resamples * self.nrows * (1 - train_size))
 
     def test_stratified(self):
         # strata size
         n = 50
-        res = ShuffleSplit(self.iris, [NaiveBayesLearner()],
-                           train_size=.5, test_size=.5,
-                           n_resamples=3, stratified=True, random_state=0)
+        res = ShuffleSplit(
+            train_size=.5, test_size=.5, n_resamples=3, stratified=True,
+            random_state=0)(self.iris, [NaiveBayesLearner()])
 
-        strata_samples = []
-        for train, test in res.indices:
-            strata_samples.append(np.count_nonzero(train < n) == n/2)
-            strata_samples.append(np.count_nonzero(train < 2 * n) == n)
-
-        self.assertTrue(all(strata_samples))
+        for fold in res.folds:
+            self.assertEqual(np.count_nonzero(res.row_indices[fold] < n), n // 2)
+            self.assertEqual(np.count_nonzero(res.row_indices[fold] < 2 * n), n)
 
     def test_not_stratified(self):
         # strata size
         n = 50
-        res = ShuffleSplit(self.iris, [NaiveBayesLearner()],
-                           train_size=.5, test_size=.5,
-                           n_resamples=3, stratified=False, random_state=0)
+        res = ShuffleSplit(
+            train_size=.5, test_size=.5, n_resamples=3, stratified=False,
+            random_state=0)(self.iris, [NaiveBayesLearner()])
 
         strata_samples = []
-        for train, test in res.indices:
-            strata_samples.append(np.count_nonzero(train < n) == n/2)
-            strata_samples.append(np.count_nonzero(train < 2 * n) == n)
+        for fold in res.folds:
+            strata_samples += [
+                np.count_nonzero(res.row_indices[fold] < n) == n // 2,
+                np.count_nonzero(res.row_indices[fold] < 2 * n) == n]
 
         self.assertTrue(not all(strata_samples))
+
+
+class TestResults(unittest.TestCase):
+    def setUp(self):
+        self.data = Table("iris")
+        self.actual = np.zeros(100)
+        self.row_indices = np.arange(100)
+        self.folds = (range(50), range(10, 60)), (range(50, 100), range(50))
+        self.learners = [MajorityLearner(), MajorityLearner()]
+        self.models = [Mock(), Mock()]
+        self.predicted = np.zeros((2, 100))
+        self.probabilities = np.zeros((2, 100, 3))
+        self.failed = [False, True]
+
+    def test_store_attributes(self):
+        res = Results(
+            self.data,
+            row_indices=self.row_indices, folds=self.folds,
+            score_by_folds=False, learners=self.learners, models=self.models,
+            failed=self.failed, actual=self.actual, predicted=self.predicted,
+            probabilities=self.probabilities, store_data=42, store_models=43)
+        self.assertIs(res.data, self.data)
+        self.assertEqual(res.nrows, 100)
+        self.assertIs(res.row_indices, self.row_indices)
+        self.assertIs(res.folds, self.folds)
+        self.assertFalse(res.score_by_folds)
+        self.assertIs(res.learners, self.learners)
+        self.assertIs(res.models, self.models)
+        self.assertIs(res.failed, self.failed)
+        self.assertIs(res.actual, self.actual)
+        self.assertIs(res.predicted, self.predicted)
+        self.assertIs(res.probabilities, self.probabilities)
+
+    def test_guess_sizes(self):
+        res = Results(self.data, actual=self.actual)
+        self.assertEqual(res.nrows, 100)
+        self.assertIsNone(res.row_indices)
+        self.assertIsNone(res.predicted)
+        self.assertIsNone(res.probabilities)
+        self.assertIsNone(res.models)
+        self.assertIsNone(res.failed)
+
+        res = Results(self.data, models=self.models)
+        self.assertIsNone(res.nrows)
+        self.assertIsNone(res.predicted)
+        self.assertIsNone(res.probabilities)
+        self.assertIs(res.models, self.models)
+        self.assertEqual(res.failed, [False, False])
+
+        res = Results(self.data, actual=self.actual, learners=self.learners)
+        self.assertIs(res.data, self.data)
+        self.assertIsNone(res.row_indices)
+        self.assertEqual(res.nrows, 100)
+        self.assertIsNone(res.folds, self.folds)
+        self.assertEqual(res.predicted.shape, (2, 100))
+        self.assertEqual(res.probabilities.shape, (2, 100, 3))
+        self.assertEqual(res.failed, [False, False])
+
+        res = Results(nrows=100, nmethods=2, nclasses=3)
+        self.assertIsNone(res.data)
+        self.assertIsNone(res.row_indices)
+        self.assertIsNone(res.folds, self.folds)
+        self.assertEqual(res.nrows, 100)
+        self.assertEqual(res.predicted.shape, (2, 100))
+        self.assertEqual(res.probabilities.shape, (2, 100, 3))
+        self.assertEqual(res.failed, [False, False])
+
+    def test_check_consistency_domain(self):
+        self.assertRaises(
+            ValueError,
+            Results, self.data, domain=Domain([], []))
+
+    def test_check_consistency_nrows(self):
+        self.assertRaises(
+            ValueError,
+            Results, nrows=10, actual=self.actual)
+        self.assertRaises(
+            ValueError,
+            Results, nrows=10, row_indices=self.row_indices)
+        self.assertRaises(
+            ValueError,
+            Results, actual=self.actual, row_indices=self.row_indices[:5])
+        self.assertRaises(
+            ValueError,
+            Results, nrows=10, predicted=self.predicted)
+        self.assertRaises(
+            ValueError,
+            Results, nrows=10, probabilities=self.probabilities)
+
+    def test_check_consistency_nclasses(self):
+        self.assertRaises(
+            ValueError,
+            Results, self.data, nclasses=10)
+        self.assertRaises(
+            ValueError,
+            Results, domain=self.data.domain, nclasses=10)
+        self.assertRaises(
+            ValueError,
+            Results,
+            nclasses=10,
+            probabilities=self.probabilities, learners=self.learners, nrows=100)
+
+        attributes = self.data.domain.attributes
+        reg_domain = Domain(attributes[:3], attributes[3])
+        self.assertRaises(
+            ValueError,
+            Results, nclasses=5, domain=self.data.domain)
+        self.assertRaises(
+            ValueError,
+            Results, nclasses=5, probabilities=self.probabilities)
+        self.assertRaises(
+            ValueError,
+            Results, nclasses=5, domain=reg_domain)
+        self.assertRaises(
+            ValueError,
+            Results, domain=reg_domain, probabilities=self.probabilities)
+
+    def test_check_consistency_nmethods(self):
+        self.assertRaises(
+            ValueError,
+            Results, nmethods=10, learners=self.learners)
+        self.assertRaises(
+            ValueError,
+            Results, nmethods=10, models=self.models)
+        self.assertRaises(
+            ValueError,
+            Results, nmethods=10, failed=self.failed)
+        self.assertRaises(
+            ValueError,
+            Results, nmethods=10, predicted=self.predicted)
+        self.assertRaises(
+            ValueError,
+            Results, nmethods=10, probabilities=self.probabilities)
+        self.assertRaises(
+            ValueError,
+            Results,
+            probabilities=self.probabilities[:1], predicted=self.predicted)
diff --git a/Orange/widgets/evaluate/tests/test_owtestlearners.py b/Orange/widgets/evaluate/tests/test_owtestlearners.py
--- a/Orange/widgets/evaluate/tests/test_owtestlearners.py
+++ b/Orange/widgets/evaluate/tests/test_owtestlearners.py
@@ -386,7 +386,7 @@ class TestHelpers(unittest.TestCase):
     def test_results_one_vs_rest(self):
         data = Table(test_filename("datasets/lenses.tab"))
         learners = [MajorityLearner()]
-        res = TestOnTestData(data[1::2], data[::2], learners=learners)
+        res = TestOnTestData()(data[1::2], data[::2], learners=learners)
         r1 = results_one_vs_rest(res, pos_index=0)
         r2 = results_one_vs_rest(res, pos_index=1)
         r3 = results_one_vs_rest(res, pos_index=2)