Repository: scikit-activeml/scikit-activeml (language: python)

Issue:
BUG: Incorrect shape handling for `annotators` parameter in query function in SingleAnnotatorWrapper
### Describe the issue:

The `annotators` parameter, expected to be of shape `(n_candidates, n_annotators)`, does not work correctly in the `query` function of class `SingleAnnotatorWrapper`.

### Reproduce the bug via a code example:

```python
import numpy as np
from sklearn.datasets import make_blobs
from skactiveml.pool import RandomSampling
from skactiveml.pool.multiannotator import SingleAnnotatorWrapper
from skactiveml.utils import majority_vote, is_labeled, is_unlabeled

if __name__ == '__main__':
    X, y_true = make_blobs(n_samples=200, random_state=0)
    X = X.astype(np.float32)
    y = np.array([y_true, y_true], dtype=float).T
    y[:100, 0] = -1
    y[100:, 1] = -1

    candidate_indicies = np.arange(50, 150)

    sa_qs = RandomSampling(random_state=0, missing_label=-1)
    ma_qs = SingleAnnotatorWrapper(sa_qs, random_state=0, missing_label=-1)

    label_available = is_labeled(y, missing_label=-1)
    A_perf = np.ones_like(y)

    query_indices = ma_qs.query(
        X=X,
        y=y,
        candidates=candidate_indicies,
        A_perf=A_perf[candidate_indicies],
        batch_size=50,
        annotators=label_available[candidate_indicies],
        n_annotators_per_sample=1,
    )
```


### Error message:

```shell
Traceback (most recent call last):
  File "/Users/chengjiaying/BachelorProjekt/Crowd_Layer/11_test.py", line 22, in <module>
    query_indices = ma_qs.query(
  File "/opt/anaconda3/envs/BachelorProjekt/lib/python3.10/site-packages/skactiveml/pool/multiannotator/_wrapper.py", line 187, in query
    ) = super()._validate_data(
  File "/opt/anaconda3/envs/BachelorProjekt/lib/python3.10/site-packages/skactiveml/base.py", line 599, in _validate_data
    check_consistent_length(X, annotators)
  File "/opt/anaconda3/envs/BachelorProjekt/lib/python3.10/site-packages/sklearn/utils/validation.py", line 397, in check_consistent_length
    raise ValueError(
ValueError: Found input variables with inconsistent numbers of samples: [200, 100]
```


### Context for the issue:

_No response_


Discussion / hints:

Fix patch:
diff --git a/skactiveml/base.py b/skactiveml/base.py
index d1b357e4..17861153 100644
--- a/skactiveml/base.py
+++ b/skactiveml/base.py
@@ -439,36 +439,40 @@ def query(
         candidates : None or array-like of shape (n_candidates), dtype=int or
             array-like of shape (n_candidates, n_features),
             optional (default=None)
-            If `candidates` is None, the samples from (X,y), for which an
-            annotator exists such that the annotator sample pair is
-            unlabeled are considered as sample candidates.
-            If `candidates` is of shape (n_candidates,) and of type int,
+            See parameter `annotators`.
+        annotators : None or array-like of shape (n_avl_annotators), dtype=int
+            or array-like of shape (n_candidates, n_annotators), optional
+        (default=None)
+            If candidate samples and annotators are not specified, i.e.,
+            `candidates=None`, `annotators=None` the unlabeled target values,
+            `y`, are the candidates annotator-sample-pairs.
+            If candidate samples and available annotators are specified:
+            The annotator-sample-pairs, for which the sample is a candidate
+            sample and the annotator is an available annotator are considered
+            as candidate annotator-sample-pairs.
+            If `candidates` is None, all samples of `X` are considered as
+            candidate samples. In this case `n_candidates` equals `len(X)`.
+            If `candidates` is of shape `(n_candidates,)` and of type int,
             `candidates` is considered as the indices of the sample candidates
-            in (X,y).
+            in `(X, y)`.
             If `candidates` is of shape (n_candidates, n_features), the
             sample candidates are directly given in `candidates` (not
             necessarily contained in `X`). This is not supported by all query
             strategies.
-        annotators : array-like of shape (n_candidates, n_annotators), optional
-        (default=None)
-            If `annotators` is None, all annotators are considered as available
-            annotators.
+            If `annotators` is `None`, all annotators are considered as
+            available annotators.
             If `annotators` is of shape (n_avl_annotators), and of type int,
             `annotators` is considered as the indices of the available
             annotators.
-            If candidate samples and available annotators are specified:
-            The annotator-sample-pairs, for which the sample is a candidate
-            sample and the annotator is an available annotator are considered
-            as candidate annotator-sample-pairs.
             If `annotators` is a boolean array of shape (n_candidates,
-            n_avl_annotators) the annotator-sample-pairs, for which the sample
+            n_annotators) the annotator-sample-pairs, for which the sample
             is a candidate sample and the boolean matrix has entry `True` are
-            considered as candidate sample pairs.
+            considered as candidate annotator-sample pairs.
         batch_size : int, optional (default=1)
-            The number of annotators sample pairs to be selected in one AL
+            The number of annotators-sample pairs to be selected in one AL
             cycle.
         return_utilities : bool, optional (default=False)
-            If true, also return the utilities based on the query strategy.
+            If True, also return the utilities based on the query strategy.
 
         Returns
         -------
@@ -478,6 +482,10 @@ def query(
             which annotator, e.g., `query_indices[:, 0]` indicates the selected
             candidate samples and `query_indices[:, 1]` indicates the
             respectively selected annotators.
+            If candidates is None or of shape (n_candidates), the indexing
+            of refers to samples in X.
+            If candidates is of shape (n_candidates, n_features), the indexing
+            refers to samples in candidates.
         utilities: numpy.ndarray of shape (batch_size, n_samples, n_annotators)
          or numpy.ndarray of shape (batch_size, n_candidates, n_annotators)
             The utilities of all candidate samples w.r.t. to the available
@@ -516,36 +524,39 @@ def _validate_data(
             that `y[i, j]` contains the label annotated by annotator `i` for
             sample `j`.
         candidates : None or array-like of shape (n_candidates), dtype=int or
-            array-like of shape (n_candidates, n_features),
-            optional (default=None)
-            If `candidates` is None, the samples from (X,y), for which an
-            annotator exists such that the annotator sample pairs is
-            unlabeled are considered as sample candidates.
-            If `candidates` is of shape (n_candidates,) and of type int,
+        array-like of shape (n_candidates, n_features), optional (default=None)
+            See annotators.
+        annotators : None or array-like of shape (n_avl_annotators), dtype=int
+        or array-like of shape (n_candidates, n_annotators), optional
+        (default=None)
+            If candidate samples and annotators are not specified, i.e.,
+            `candidates=None`, `annotators=None` the unlabeled target values,
+            `y`, are the candidates annotator-sample-pairs.
+            If candidate samples and available annotators are specified:
+            The annotator-sample-pairs, for which the sample is a candidate
+            sample and the annotator is an available annotator are considered
+            as candidate annotator-sample-pairs.
+            If `candidates` is None, all samples of `X` are considered as
+            candidate samples. In this case `n_candidates` equals `len(X)`.
+            If `candidates` is of shape `(n_candidates,)` and of type int,
             `candidates` is considered as the indices of the sample candidates
-            in (X,y).
-            If `candidates` is of shape (n_candidates, n_features), the
+            in `(X, y)`.
+            If `candidates` is of shape `(n_candidates, n_features)`, the
             sample candidates are directly given in `candidates` (not
             necessarily contained in `X`). This is not supported by all query
             strategies.
-        annotators : array-like of shape (n_candidates, n_annotators), optional
-        (default=None)
-            If `annotators` is None, all annotators are considered as available
-            annotators.
-            If `annotators` is of shape (n_avl_annotators), and of type int,
+            If `annotators` is `None`, all annotators are considered as
+            available annotators.
+            If `annotators` is of shape `(n_avl_annotators)`, and of type int,
             `annotators` is considered as the indices of the available
             annotators.
-            If candidate samples and available annotators are specified:
-            The annotator-sample-pairs, for which the sample is a candidate
-            sample and the annotator is an available annotator are considered
-            as candidate annotator-sample-pairs.
-            If `annotators` is a boolean array of shape (n_candidates,
-            n_avl_annotators) the annotator-sample-pairs, for which the sample
+            If `annotators` is a boolean array of shape `(n_candidates,
+            n_annotators)` the annotator-sample-pairs, for which the sample
             is a candidate sample and the boolean matrix has entry `True` are
-            considered as candidate sample pairs.
+            considered as candidate annotator-sample-pairs.
         batch_size : int or string, optional (default=1)
             The number of annotators sample pairs to be selected in one AL
-            cycle. If `adaptive = True` `batch_size = 'adaptive'` is allowed.
+            cycle. If `adaptive=True`, `batch_size='adaptive'` is allowed.
         return_utilities : bool
             If true, also return the utilities based on the query strategy.
         reset : bool, default=True
@@ -595,30 +606,24 @@ def _validate_data(
                 annotators = check_indices(annotators, y, dim=1)
             elif annotators.ndim == 2:
                 annotators = check_array(annotators, dtype=bool)
-                if candidates is None or candidates.ndim == 1:
+                if candidates is None:
                     check_consistent_length(X, annotators)
                 else:
                     check_consistent_length(candidates, annotators)
                 check_consistent_length(y.T, annotators.T)
             else:
                 raise ValueError(
-                    "`annotators` must be either None, 1d or 2d " "array-like."
+                    "`annotators` must be either None, 1d or 2d array-like."
                 )
 
         if annotators is None:
             if candidates is None:
                 n_candidate_pairs = int(np.sum(unlabeled_pairs))
-            elif candidates.ndim == 1:
-                n_candidate_pairs = len(candidates) * len(y.T)
             else:
                 n_candidate_pairs = len(candidates) * len(y.T)
         elif annotators.ndim == 1:
             if candidates is None:
-                n_candidate_pairs = int(np.sum(unlabeled_pairs[:, annotators]))
-            elif candidates.ndim == 1:
-                n_candidate_pairs = int(
-                    np.sum(unlabeled_pairs[candidates][:, annotators])
-                )
+                n_candidate_pairs = len(X) * len(annotators)
             else:
                 n_candidate_pairs = len(candidates) * len(annotators)
         else:
@@ -649,48 +654,52 @@ def _transform_cand_annot(
         candidates : None or array-like of shape (n_candidates), dtype=int or
             array-like of shape (n_candidates, n_features),
             optional (default=None)
-            If `candidates` is None, the samples from (X,y), for which an
-            annotator exists such that the annotator sample pairs is
-            unlabeled are considered as sample candidates.
+            See annotators.
+        annotators : None or array-like of shape (n_avl_annotators), dtype=int
+        or array-like of shape (n_candidates, n_annotators), optional
+        (default=None)
+            If candidate samples and annotators are not specified, i.e.,
+            `candidates=None`, `annotators=None` the unlabeled target values,
+            `y`, are the candidates annotator-sample-pairs.
+            If candidate samples and available annotators are specified:
+            The annotator-sample-pairs, for which the sample is a candidate
+            sample and the annotator is an available annotator are considered
+            as candidate annotator-sample-pairs.
+            If `candidates` is `None`, all samples of `X` are considered as
+            candidate samples. In this case `n_candidates` equals `len(X)`.
             If `candidates` is of shape (n_candidates,) and of type int,
             `candidates` is considered as the indices of the sample candidates
-            in (X,y).
-            If `candidates` is of shape (n_candidates, n_features), the
+            in `(X, y)`.
+            If `candidates` is of shape `(n_candidates, n_features)`, the
             sample candidates are directly given in `candidates` (not
             necessarily contained in `X`). This is not supported by all query
             strategies.
-        annotators : array-like of shape (n_candidates, n_annotators), optional
-        (default=None)
-            If `annotators` is None, all annotators are considered as available
-            annotators.
+            If `annotators` is `None`, all annotators are considered as
+            available annotators.
             If `annotators` is of shape (n_avl_annotators), and of type int,
             `annotators` is considered as the indices of the available
             annotators.
-            If candidate samples and available annotators are specified:
-            The annotator-sample-pairs, for which the sample is a candidate
-            sample and the annotator is an available annotator are considered
-            as candidate annotator-sample-pairs.
-            If `annotators` is a boolean array of shape (n_candidates,
-            n_avl_annotators) the annotator-sample-pairs, for which the sample
+            If `annotators` is a boolean array of shape `(n_candidates,
+            n_annotators)` the annotator-sample-pairs, for which the sample
             is a candidate sample and the boolean matrix has entry `True` are
-            considered as candidate sample pairs.
+            considered as candidate annotator-sample-pairs.
         X : np.ndarray of shape (n_samples, n_features)
             Checked training data set.
         y : np.ndarray of shape (n_samples,)
             Checked labels of the training data set.
         enforce_mapping : bool, optional (default=False)
             If `True`, an exception is raised when no exact mapping can be
-            determined (i.e., `mapping` is None).
+            determined (i.e., `mapping` is `None`).
 
         Returns
         -------
-        candidates : np.ndarray of shape (n_candidates, n_features)
+        candidates : np.ndarray of shape (n_selectable_candidates, n_features)
             Candidate samples from which the strategy can query the label.
-        mapping : np.ndarray of shape (n_candidates) or None
+        mapping : np.ndarray of shape (n_selectable_candidates) or None
             Index array that maps `candidates` to `X`
             (`candidates = X[mapping]`).
-        A_cand : np.ndarray of shape(n_candidates, n_annotators)
-            Available annotator sample pair with respect to `candidates`.
+        A_cand : np.ndarray of shape(n_selectable_candidates, n_annotators)
+            Available annotator-sample-pairs with respect to `candidates`.
         """
         unlbd_pairs = is_unlabeled(y, self.missing_label_)
         unlbd_sample_indices = np.argwhere(
@@ -698,6 +707,7 @@ def _transform_cand_annot(
         ).flatten()
         n_annotators = y.shape[1]
 
+        # if mapping does not exist
         if candidates is not None and candidates.ndim == 2:
             n_candidates = len(candidates)
             if annotators is None:
@@ -716,27 +726,27 @@ def _transform_cand_annot(
                 )
             else:
                 return candidates, None, A_cand
-
+        # mapping exists
         if candidates is None:
-            candidates = unlbd_sample_indices
-            only_candidates = False
-        elif annotators is not None:
-            candidates = np.intersect1d(candidates, unlbd_sample_indices)
-            only_candidates = False
-        else:
-            only_candidates = True
-
-        if only_candidates:
-            A_cand = np.full((len(candidates), n_annotators), True)
-        elif annotators is None:
-            A_cand = unlbd_pairs[candidates, :]
-        elif annotators.ndim == 1:
-            available_pairs = np.full_like(y, False, dtype=bool)
-            available_pairs[:, annotators] = True
-            A_cand = (unlbd_pairs & available_pairs)[candidates, :]
-        else:
-            A_cand = annotators
-
+            if annotators is None:
+                candidates = unlbd_sample_indices
+                A_cand = unlbd_pairs[unlbd_sample_indices]
+            elif annotators.ndim == 1:
+                candidates = np.arange(len(X), dtype=int)
+                A_cand = np.full_like(y, False)
+                A_cand[:, annotators] = True
+            else:
+                candidates = np.arange(len(X), dtype=int)
+                A_cand = annotators
+        else:  # candidates indices array
+            if annotators is None:
+                A_cand = np.full((len(candidates), y.shape[1]), True)
+            elif annotators.ndim == 1:
+                A_cand = np.full((len(candidates), y.shape[1]), False)
+                A_cand[:, annotators] = True
+            else:
+                candidates = candidates
+                A_cand = annotators
         return X[candidates], candidates, A_cand
 
 
diff --git a/skactiveml/pool/_epistemic_uncertainty_sampling.py b/skactiveml/pool/_epistemic_uncertainty_sampling.py
index 27aa9d85..22c3a8f0 100644
--- a/skactiveml/pool/_epistemic_uncertainty_sampling.py
+++ b/skactiveml/pool/_epistemic_uncertainty_sampling.py
@@ -11,7 +11,7 @@
 from scipy.optimize import minimize_scalar, minimize, LinearConstraint
 from sklearn import clone
 from sklearn.linear_model import LogisticRegression
-from sklearn.utils.extmath import safe_sparse_dot, log_logistic
+from sklearn.utils.extmath import safe_sparse_dot
 
 from ..base import SingleAnnotatorPoolQueryStrategy, SkactivemlClassifier
 from ..classifier import SklearnClassifier, ParzenWindowClassifier
@@ -661,7 +661,7 @@ def _logistic_loss(w, X, y, alpha, sample_weight=None):
         sample_weight = np.ones(y.shape[0])
 
     # Logistic loss is the negative of the log of the logistic function.
-    out = -np.sum(sample_weight * log_logistic(yz))
+    out = -np.sum(sample_weight * (-np.logaddexp(0, -yz)))
     out += 0.5 * alpha * np.dot(w, w)
     return out
 
diff --git a/skactiveml/pool/_four_ds.py b/skactiveml/pool/_four_ds.py
index 7d67f766..c0f8d5cb 100644
--- a/skactiveml/pool/_four_ds.py
+++ b/skactiveml/pool/_four_ds.py
@@ -211,7 +211,7 @@ def query(
         )
         query_indices_cand[0] = rand_argmax(
             utilities_cand[0], self.random_state_
-        )
+        )[0]
         is_selected = np.zeros(len(X_cand), dtype=bool)
         is_selected[query_indices_cand[0]] = True
 
@@ -258,7 +258,7 @@ def query(
                 utilities_cand[i, is_selected] = np.nan
                 query_indices_cand[i] = rand_argmax(
                     utilities_cand[i], self.random_state_
-                )
+                )[0]
                 is_selected[query_indices_cand[i]] = True
 
         # Remapping of utilities and query indices if required.
diff --git a/skactiveml/pool/_greedy_sampling.py b/skactiveml/pool/_greedy_sampling.py
index cb4e8374..30452d47 100644
--- a/skactiveml/pool/_greedy_sampling.py
+++ b/skactiveml/pool/_greedy_sampling.py
@@ -430,7 +430,7 @@ def _greedy_sampling(
         utilities[i, not_selected_candidates] = util
 
         idx = rand_argmax(util, random_state=random_state)
-        query_indices[i] = not_selected_candidates[idx]
+        query_indices[i] = not_selected_candidates[idx][0]
         distances[:, candidate_indices[idx]] = _measure_distance(
             candidate_indices[idx], **dist_dict
         )
diff --git a/skactiveml/pool/multiannotator/_interval_estimation_threshold.py b/skactiveml/pool/multiannotator/_interval_estimation_threshold.py
index 0b22aec3..7c7e1f99 100644
--- a/skactiveml/pool/multiannotator/_interval_estimation_threshold.py
+++ b/skactiveml/pool/multiannotator/_interval_estimation_threshold.py
@@ -45,8 +45,6 @@ class IntervalEstimationAnnotModel(BaseEstimator, AnnotatorModelMixin):
 
     Attributes
     ----------
-    classes_: array-like, shape (n_classes)
-        Holds the label for each class.
     n_annotators_: int
         Number of annotators.
     A_perf_ : ndarray, shape (n_annotators, 3)
@@ -255,33 +253,35 @@ def query(
         candidates : None or array-like of shape (n_candidates), dtype=int or
             array-like of shape (n_candidates, n_features),
             optional (default=None)
-            If `candidates` is None, the samples from (X,y), for which an
-            annotator exists such that the annotator sample pairs is
-            unlabeled are considered as sample candidates.
-            If `candidates` is of shape (n_candidates,) and of type int,
+            See annotators.
+        annotators : None or array-like of shape (n_avl_annotators), dtype=int
+        or array-like of shape (n_candidates, n_annotators), optional
+        (default=None)
+            If candidate samples and annotators are not specified, i.e.,
+            `candidates=None`, `annotators=None` the unlabeled target values,
+            `y`, are the candidates annotator-sample-pairs.
+            If candidate samples and available annotators are specified:
+            The annotator-sample-pairs, for which the sample is a candidate
+            sample and the annotator is an available annotator are considered
+            as candidate annotator-sample-pairs.
+            If `candidates` is None, all samples of `X` are considered as
+            candidate samples. In this case `n_candidates` equals `len(X)`.
+            If `candidates` is of shape `(n_candidates,)` and of type int,
             `candidates` is considered as the indices of the sample candidates
-            in (X,y).
-            If `candidates` is of shape (n_candidates, n_features), the
+            in `(X, y)`.
+            If `candidates` is of shape `(n_candidates, n_features)`, the
             sample candidates are directly given in `candidates` (not
-            necessarily contained in `X`).
-        annotators : array-like, shape (n_candidates, n_annotators), optional
-        (default=None)
-            If `annotators` is None, all annotators are considered as available
-            annotators.
-            If `annotators` is of shape (n_avl_annotators) and of type int,
+            necessarily contained in `X`). This is not supported by all query
+            strategies.
+            If `annotators` is `None`, all annotators are considered as
+            available annotators.
+            If `annotators` is of shape `(n_avl_annotators)`, and of type int,
             `annotators` is considered as the indices of the available
             annotators.
-            If candidate samples and available annotators are specified:
-            The annotator sample pairs, for which the sample is a candidate
-            sample and the annotator is an available annotator are considered
-            as candidate annotator sample pairs.
-            If `annotators` is None and `candidates` is of shape
-            (n_candidates,), all annotator sample pairs, for which the sample
-            is indexed by `candidates` are considered as candidate annotator
-            sample pairs. If `annotators` is a boolean array of shape
-            (n_candidates, n_avl_annotators) the annotator sample pairs, for
-            which the sample is a candidate sample and the boolean matrix has
-            entry `True` are considered as candidate sample pairs.
+            If `annotators` is a boolean array of shape `(n_candidates,
+            n_annotators)` the annotator-sample-pairs, for which the sample
+            is a candidate sample and the boolean matrix has entry `True` are
+            considered as candidate annotator-sample-pairs.
         sample_weight : array-like, (n_samples, n_annotators), optional
         (default=None)
             It contains the weights of the training samples' class labels.
diff --git a/skactiveml/pool/multiannotator/_wrapper.py b/skactiveml/pool/multiannotator/_wrapper.py
index 5de3719e..908a488a 100644
--- a/skactiveml/pool/multiannotator/_wrapper.py
+++ b/skactiveml/pool/multiannotator/_wrapper.py
@@ -89,50 +89,51 @@ def query(
         candidates : None or array-like of shape (n_candidates), dtype=int or
             array-like of shape (n_candidates, n_features),
             optional (default=None)
-            If `candidates` is None, the samples from (X,y), for which an
-            annotator exists such that the annotator sample pairs is
-            unlabeled are considered as sample candidates.
-            If `candidates` is of shape (n_candidates) and of type int,
-            candidates is considered as the indices of the sample candidates in
-            (X,y).
-            If `candidates` is of shape (n_candidates, n_features), the
-            sample candidates are directly given in candidates (not necessarily
-            contained in X). This is not supported by all query strategies.
-        annotators : array-like of shape (n_candidates, n_annotators), optional
+            See annotators.
+        annotators : None or array-like of shape (n_avl_annotators), dtype=int
+        or array-like of shape (n_candidates, n_annotators), optional
         (default=None)
-            If `annotators` is None, all annotators are considered as available
-            annotators.
-            If `annotators` is of shape (n_avl_annotators) and of type int,
-            `annotators` is considered as the indices of the available
-            annotators.
+            If candidate samples and annotators are not specified, i.e.,
+            `candidates=None`, `annotators=None` the unlabeled target values,
+            `y`, are the candidates annotator-sample-pairs.
             If candidate samples and available annotators are specified:
-            The annotator sample pairs, for which the sample is a candidate
+            The annotator-sample-pairs, for which the sample is a candidate
             sample and the annotator is an available annotator are considered
             as candidate annotator-sample-pairs.
-            If `annotators` is None and `candidates` is of shape
-            (n_candidates), all annotator sample pairs, for which the sample is
-            indexed by `candidates` are considered as candidate
-            annotator-sample-pairs.
-            If `annotators` is a boolean array of shape (n_candidates,
-            n_avl_annotators) the annotator sample pairs, for which the sample
+            If `candidates` is None, all samples of `X` are considered as
+            candidate samples. In this case `n_candidates` equals `len(X)`.
+            If `candidates` is of shape `(n_candidates,)` and of type int,
+            `candidates` is considered as the indices of the sample candidates
+            in `(X, y)`.
+            If `candidates` is of shape `(n_candidates, n_features)`, the
+            sample candidates are directly given in `candidates` (not
+            necessarily contained in `X`). This is not supported by all query
+            strategies.
+            If `annotators` is `None`, all annotators are considered as
+            available annotators.
+            If `annotators` is of shape `(n_avl_annotators)`, and of type int,
+            `annotators` is considered as the indices of the available
+            annotators.
+            If `annotators` is a boolean array of shape `(n_candidates,
+            n_annotators)` the annotator-sample-pairs, for which the sample
             is a candidate sample and the boolean matrix has entry `True` are
-            considered as candidate sample pairs.
+            considered as candidate annotator-sample-pairs.
         batch_size : int, optional (default=1)
             The number of annotators sample pairs to be selected in one AL
             cycle.
-        A_perf : array-like, shape (n_samples, n_annotators) or
-                  (n_annotators,) optional (default=None)
+        A_perf : array-like, shape (n_annotators,) or
+        (n_candidates, n_annotators), optional (default=None)
             The performance based ranking of each annotator.
-            1.) If `A_perf` is of shape (n_samples, n_annotators) for each
-             sample `i` the value-annotators pair `(i, j)` is chosen
-             over the pair `(i, k)` if `A_perf[i, j]` is greater or
-             equal to `A_perf[i, k]`.
+            1.) If `A_perf` is of shape (n_candidates, n_annotators) for each
+            sample `i` the value-annotators pair `(i, j)` is chosen
+            over the pair `(i, k)` if `A_perf[i, j]` is greater or
+            equal to `A_perf[i, k]`.
             2.) If `A_perf` is of shape (n_annotators,) for each sample
-             `i` the value-annotators pair `(i, j)` is chosen over
-              the pair `(i, k)` if `A_perf[j]` is greater or
-             equal to `A_perf[k]`.
+            `i` the value-annotators pair `(i, j)` is chosen over
+            the pair `(i, k)` if `A_perf[j]` is greater or
+            equal to `A_perf[k]`.
             3.) If `A_perf` is None, the annotators are chosen at random, with
-             a different distribution for each sample.
+            a different distribution for each sample.
         return_utilities : bool, optional (default=False)
             If true, also returns the utilities based on the query strategy.
         n_annotators_per_sample : int, array-like, optional (default=1)
@@ -142,7 +143,7 @@ def query(
             candidate sample, selected by the query_strategy.
             `Preferably` in this case means depending on how many annotators
             can be assigned to a given candidate sample and how many
-            annotator-sample pairs should be assigned considering the
+            annotator-sample-pairs should be assigned considering the
             `batch_size`.
             If `n_annotators_per_sample` is an int array, the values of the
             array are interpreted as follows. The value at the i-th index
@@ -161,7 +162,7 @@ def query(
         -------
         query_indices : np.ndarray of shape (batchsize, 2)
             The query_indices indicate which candidate sample pairs are to be
-            queried is, i. e. which candidate sample is to be annotated by
+            queried is, i.e., which candidate sample is to be annotated by
             which annotator, e.g., `query_indices[:, 0]` indicates the selected
             candidate samples and `query_indices[:, 1]` indicates the
             respectively selected annotators.
@@ -204,6 +205,15 @@ def query(
         check_type(
             self.strategy, "self.strategy", SingleAnnotatorPoolQueryStrategy
         )
+        if self.strategy.missing_label != self.missing_label and not (
+            np.isnan(self.strategy.missing_label)
+            & np.isnan(self.missing_label)
+        ):
+            raise ValueError(
+                f"`self.missing_label` must equal `self.strategy.missing_label`, but"
+                f"`self.missing_label` equals {self.missing_label} and"
+                f"`self.strategy.missing_label` equals {self.strategy.missing_label}."
+            )
 
         # aggregate y
         if self.y_aggregate is None:
@@ -236,8 +246,9 @@ def query(
 
         y_sq = y_aggregate(y)
 
-        n_candidates = X_cand.shape[0]
-        n_annotators = A_cand.shape[1]
+        n_selectable_candidates = len(X_cand)
+        n_candidates = len(candidates) if candidates is not None else len(X)
+        n_annotators = y.shape[1]
         n_samples = X.shape[0]
 
         batch_size_sq = min(batch_size, X_cand.shape[0])
@@ -282,7 +293,7 @@ def query(
         # check A_perf and set annotator_utilities
         if A_perf is None:
             annotator_utilities = random_state.rand(
-                1, n_candidates, n_annotators
+                1, n_selectable_candidates, n_annotators
             ).repeat(batch_size_sq, axis=0)
         elif _is_arraylike(A_perf):
             A_perf = check_array(A_perf, ensure_2d=False)
@@ -300,16 +311,18 @@ def query(
                 annotator_utilities = A_perf[np.newaxis, :, :].repeat(
                     batch_size_sq, axis=0
                 )
+                if candidates is None:
+                    annotator_utilities = annotator_utilities[:, mapping, :]
             elif A_perf.shape == (n_annotators,):
                 annotator_utilities = (
                     A_perf[np.newaxis, np.newaxis, :]
-                    .repeat(n_candidates, axis=1)
+                    .repeat(n_selectable_candidates, axis=1)
                     .repeat(batch_size_sq, axis=0)
                 )
             else:
                 raise ValueError(
                     f"`A_perf` is of shape {A_perf.shape}, but must be of "
-                    f"shape ({n_candidates}, {n_annotators}) or of shape "
+                    f"shape ({n_selectable_candidates}, {n_annotators}) or of shape "
                     f"({n_annotators},)."
                 )
         else:
@@ -319,7 +332,7 @@ def query(
             )
 
         candidates_sq = mapping if mapping is not None else X_cand
-        re_val = self.strategy.query(
+        _, w_utilities = self.strategy.query(
             X=X,
             y=y_sq,
             candidates=candidates_sq,
@@ -328,8 +341,6 @@ def query(
             return_utilities=True,
         )
 
-        single_query_indices, w_utilities = re_val
-
         if mapping is None:
             sample_utilities = w_utilities
         else:
@@ -375,12 +386,10 @@ def _query_annotators(
         n_annotators = A_cand.shape[1]
         n_samples = A_cand.shape[0]
 
-        re_val = self._get_order_preserving_s_query(
+        s_indices, s_utilities = self._get_order_preserving_s_query(
             A_cand, sample_utilities, annotator_utilities
         )
 
-        s_indices, s_utilities = re_val
-
         n_as_annotators = self._n_to_assign_annotators(
             batch_size, A_cand, s_indices, pref_n_annotators
         )
@@ -388,11 +397,10 @@ def _query_annotators(
         utilities = np.zeros((batch_size, n_samples, n_annotators))
         query_indices = np.zeros((batch_size, 2), dtype=int)
 
-        batch_index = 0  # actual batch index
         annotator_ps = 0  # current annotators per sample
         sample_index = 0  # sample batch index
 
-        while batch_index < batch_size:
+        for batch_index in range(batch_size):  # actual batch index
             utilities[batch_index] = s_utilities[sample_index]
             query_indices[batch_index] = rand_argmax(
                 utilities[batch_index], random_state=random_state
@@ -402,7 +410,6 @@ def _query_annotators(
                 :, query_indices[batch_index, 0], query_indices[batch_index, 1]
             ] = np.nan
 
-            batch_index += 1
             annotator_ps += 1
             if annotator_ps >= n_as_annotators[sample_index]:
                 sample_index += 1
@@ -431,7 +438,7 @@ def _get_order_preserving_s_query(
 
         candidate_utilities[nan_indices[:, 0], nan_indices[:, 1]] = np.nan
 
-        annotator_utilities[:, A == 0] = np.nan
+        annotator_utilities[:, ~A] = np.nan
 
         # combine utilities by addition
         utilities = candidate_utilities[:, :, np.newaxis] + annotator_utilities
@@ -454,5 +461,7 @@ def _n_to_assign_annotators(batch_size, A, s_indices, pref_n_annotators):
             )
 
             n_annotator_sample_pairs = np.sum(annot_per_sample)
+            if n_annotator_sample_pairs >= batch_size:
+                break
 
         return annot_per_sample
diff --git a/skactiveml/visualization/_misc.py b/skactiveml/visualization/_misc.py
index a087828e..f089dec0 100644
--- a/skactiveml/visualization/_misc.py
+++ b/skactiveml/visualization/_misc.py
@@ -41,7 +41,7 @@ def mesh(bound, res):
 
 def _get_cmap(cmap):
     if isinstance(cmap, str):
-        cmap = plt.cm.get_cmap(cmap)
+        cmap = plt.colormaps.get_cmap(cmap)
     check_type(cmap, "cmap", Colormap, str)
     return cmap
 


Test patch:
diff --git a/skactiveml/pool/multiannotator/tests/test_wrapper.py b/skactiveml/pool/multiannotator/tests/test_wrapper.py
index 96e8616a..43863cc5 100644
--- a/skactiveml/pool/multiannotator/tests/test_wrapper.py
+++ b/skactiveml/pool/multiannotator/tests/test_wrapper.py
@@ -1,11 +1,13 @@
 import unittest
 
 import numpy as np
+from sklearn.datasets import make_blobs
 from sklearn.gaussian_process import GaussianProcessClassifier
 
 from skactiveml.classifier import SklearnClassifier, MixtureModelClassifier
 from skactiveml.pool import UncertaintySampling, RandomSampling
 from skactiveml.pool.multiannotator._wrapper import SingleAnnotatorWrapper
+from skactiveml.utils import is_labeled
 from skactiveml.utils import majority_vote, MISSING_LABEL, is_unlabeled
 
 
@@ -36,28 +38,43 @@ def setUp(self):
     def test_init_param_strategy(self):
         wrapper = SingleAnnotatorWrapper(MixtureModelClassifier())
 
-        query_kwargs = {"X": self.X, "y": self.y}
         self.assertRaises(
             TypeError,
             wrapper.query,
-            self.X_cand,
-            A_cand=self.A_cand,
-            **query_kwargs,
+            self.X,
+            self.y,
+            candidates=self.X_cand,
+            annotators=self.A_cand,
         )
 
         wrapper = SingleAnnotatorWrapper(0)
 
-        query_kwargs = {"X": self.X, "y": self.y}
         self.assertRaises(
             TypeError,
             wrapper.query,
-            self.X_cand,
-            A_cand=self.A_cand,
-            **query_kwargs,
+            self.X,
+            self.y,
+            candidates=self.X_cand,
+            annotators=self.A_cand,
+        )
+
+        strategy = RandomSampling(
+            random_state=self.random_state, missing_label=-1
+        )
+        wrapper = SingleAnnotatorWrapper(
+            strategy=strategy, random_state=self.random_state, missing_label=0
+        )
+        self.assertRaises(
+            ValueError,
+            wrapper.query,
+            self.X,
+            self.y,
+            candidates=self.X_cand,
+            annotators=self.A_cand,
         )
 
     def test_init_param_y_aggregate(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(
             random, y_aggregate="string", random_state=self.random_state
         )
@@ -74,7 +91,7 @@ def test_init_param_y_aggregate(self):
 
         dummy_function = lambda x, y: majority_vote(x)
 
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(
             random, y_aggregate=dummy_function, random_state=self.random_state
         )
@@ -90,13 +107,13 @@ def test_init_param_y_aggregate(self):
         )
 
     def test_init_param_random_state(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(random, random_state="string")
 
         self.assertRaises(ValueError, wrapper.query, self.X, self.y)
 
     def test_query_param_candidates(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         X_cand = [1, 0, 2, 4]
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
@@ -200,7 +217,7 @@ def test_query_param_X(self):
         )
 
     def test_query_param_annotators(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
         )
@@ -216,7 +233,7 @@ def test_query_param_annotators(self):
         )
 
     def test_query_param_batch_size(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
         )
@@ -225,7 +242,7 @@ def test_query_param_batch_size(self):
         )
 
     def test_query_param_return_utilities(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
         )
@@ -240,7 +257,7 @@ def test_query_param_return_utilities(self):
         )
 
     def test_query_param_n_annotators_per_sample(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
         )
@@ -265,7 +282,7 @@ def test_query_param_n_annotators_per_sample(self):
         )
 
     def test_query_param_A_perf(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
         )
@@ -360,7 +377,7 @@ def y_aggregate(y_t):
         self.check_availability(best_cand_indices, A)
 
     def test_query_one_annotator_per_sample_batch_size_five(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
 
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
@@ -382,7 +399,7 @@ def test_query_one_annotator_per_sample_batch_size_five(self):
         self.check_availability(best_cand_indices, self.A_cand)
 
     def test_query_batch_size_too_large(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
 
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
@@ -398,7 +415,7 @@ def test_query_batch_size_too_large(self):
         self.check_max(best_cand_indices, utilities)
 
     def test_query_three_n_annotators_per_sample_batch_size_five(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
 
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
@@ -423,7 +440,7 @@ def test_query_three_n_annotators_per_sample_batch_size_five(self):
     def test_query_three_n_annotators_per_sample_batch_size_five_mismatch(
         self,
     ):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
 
         X_cand = np.array([[7, 1], [9, 1]])
 
@@ -450,7 +467,7 @@ def test_query_three_n_annotators_per_sample_batch_size_five_mismatch(
         self.check_availability(best_cand_indices, A_cand)
 
     def test_query_varying_n_annotators_per_sample_batch_size_five(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
 
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
@@ -476,7 +493,7 @@ def test_query_varying_n_annotators_per_sample_batch_size_five(self):
         self.check_max(best_cand_indices, utilities)
 
     def test_query_per_sample_too_large(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
         )
@@ -498,7 +515,7 @@ def test_query_per_sample_too_large(self):
         self.check_max(best_cand_indices, utilities)
 
     def test_query_unavailable_annotators(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
 
         X_cand = np.array([[7, 1], [9, 1], [3, 5], [2, 7]])
 
@@ -531,38 +548,62 @@ def test_query_unavailable_annotators(self):
         self.check_availability(best_cand_indices, A_cand)
 
     def test_query_custom_annotator_special_preference(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
 
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
         )
 
         X_cand = np.array([[7, 1], [9, 1]])
+        A_perf = np.array([[1, 2, 3], [3, 2, 1]])
+
+        for candidates in [X_cand, np.array([0, 1])]:
+            best_cand_indices, utilities = wrapper.query(
+                self.X,
+                self.y,
+                candidates=candidates,
+                batch_size=6,
+                n_annotators_per_sample=3,
+                A_perf=A_perf,
+                return_utilities=True,
+            )
 
+            # assert the utilities fit A_perf
+            self.assertFalse(np.any(utilities[:, 0, 2] < utilities[:, 0, 1]))
+            self.assertFalse(np.any(utilities[:, 0, 1] < utilities[:, 0, 0]))
+
+            self.assertFalse(np.any(utilities[:, 1, 0] < utilities[:, 1, 1]))
+            self.assertFalse(np.any(utilities[:, 1, 1] < utilities[:, 1, 2]))
+
+            self.check_max(best_cand_indices, utilities)
+
+        X = np.array([[7, 1], [9, 1]])
+        y = np.array(
+            [
+                [MISSING_LABEL, 0, MISSING_LABEL],
+                [MISSING_LABEL, MISSING_LABEL, MISSING_LABEL],
+            ]
+        )
         A_perf = np.array([[1, 2, 3], [3, 2, 1]])
 
-        re_val = wrapper.query(
-            self.X,
-            self.y,
-            candidates=X_cand,
-            batch_size=6,
+        best_cand_indices, utilities = wrapper.query(
+            X,
+            y,
+            batch_size=5,
             n_annotators_per_sample=3,
             A_perf=A_perf,
             return_utilities=True,
         )
 
-        best_cand_indices, utilities = re_val
         # assert the utilities fit A_perf
         self.assertFalse(np.any(utilities[:, 0, 2] < utilities[:, 0, 1]))
         self.assertFalse(np.any(utilities[:, 0, 1] < utilities[:, 0, 0]))
-
         self.assertFalse(np.any(utilities[:, 1, 0] < utilities[:, 1, 1]))
         self.assertFalse(np.any(utilities[:, 1, 1] < utilities[:, 1, 2]))
-
         self.check_max(best_cand_indices, utilities)
 
     def test_query_custom_annotator_general_equal_preference(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
 
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
@@ -599,7 +640,7 @@ def test_query_custom_annotator_general_equal_preference(self):
         )
 
     def test_query_indexed_annotator_sample_candidates(self):
-        random = RandomSampling(self.random_state)
+        random = RandomSampling(random_state=self.random_state)
         wrapper = SingleAnnotatorWrapper(
             random, random_state=self.random_state
         )
@@ -625,6 +666,38 @@ def test_query_indexed_annotator_sample_candidates(self):
         self.assertEqual((4, 2, 3), utilities.shape)
         self.check_max(best_cand_indices, utilities)
 
+    def test_query_external_annotation_matrix(self):
+        X, y_true = make_blobs(n_samples=200, random_state=0)
+        X = X.astype(np.float32)
+        y = np.array([y_true, y_true], dtype=float).T
+        y[:100, 0] = -1
+        y[100:, 1] = -1
+
+        candidate_indicies = np.arange(50, 150)
+
+        sa_qs = RandomSampling(random_state=0, missing_label=-1)
+        ma_qs = SingleAnnotatorWrapper(sa_qs, random_state=0, missing_label=-1)
+
+        label_available = is_labeled(y, missing_label=-1)
+        A_perf = np.ones_like(y)
+
+        query_indices = ma_qs.query(
+            X=X,
+            y=y,
+            candidates=candidate_indicies,
+            A_perf=A_perf[candidate_indicies],
+            batch_size=50,
+            annotators=label_available[candidate_indicies],
+            n_annotators_per_sample=1,
+        )
+        self.assertEqual((50, 2), query_indices.shape)
+        # candiate indices contained in candidate_indices
+        self.assertTrue(np.isin(query_indices[:, 0], candidate_indicies).all())
+        # pair indices contained in label_available
+        self.assertTrue(
+            np.all(label_available[query_indices[:, 0], query_indices[:, 1]])
+        )
+
     def check_availability(self, best_cand_indices, A_cand):
         best_value_indices = best_cand_indices[:, 0]
         best_annotator_indices = best_cand_indices[:, 1]
diff --git a/skactiveml/tests/test_base.py b/skactiveml/tests/test_base.py
index 0a90ffa2..d4fa6574 100644
--- a/skactiveml/tests/test_base.py
+++ b/skactiveml/tests/test_base.py
@@ -1,4 +1,5 @@
 import unittest
+from itertools import product
 from unittest.mock import patch
 
 import numpy as np
@@ -16,7 +17,7 @@
     ProbabilisticRegressor,
 )
 from skactiveml.exceptions import MappingError
-from skactiveml.utils import MISSING_LABEL
+from skactiveml.utils import MISSING_LABEL, is_unlabeled
 
 
 class QueryStrategyTest(unittest.TestCase):
@@ -90,8 +91,7 @@ def test_query(self):
             ),
         )
 
-    def test__transform_cand_annot(self):
-        # TODO separate _validate_data() testing
+    def test__validate_data(self):
         self.assertRaises(
             ValueError,
             self.qs._validate_data,
@@ -105,6 +105,39 @@ def test__transform_cand_annot(self):
             return_utilities=False,
         )
 
+        X = np.array([[1, 2], [0, 1]])
+        y = np.array([[1, MISSING_LABEL], [2, 3]])
+        candidates_values = [
+            None,
+            np.array([0, 1]),
+            np.array([[3, 4], [0, 1]]),
+        ]
+        annotators_values = [
+            None,
+            np.array([0]),
+            np.array([[False, True], [True, True]]),
+        ]
+
+        batch_size_initial = 4
+        batch_sizes_expected = [[1, 2, 3], [4, 2, 3], [4, 2, 3]]
+
+        for (i, candidates), (j, annotators) in product(
+            enumerate(candidates_values), enumerate(annotators_values)
+        ):
+            X, y, candidates, annotators, batch_size, return_utilities = (
+                self.qs._validate_data(
+                    candidates=candidates,
+                    annotators=annotators,
+                    X=X,
+                    y=y,
+                    batch_size=batch_size_initial,
+                    return_utilities=False,
+                )
+            )
+            self.assertEqual(batch_sizes_expected[i][j], batch_size)
+
+    def test__transform_cand_annot(self):
+
         self.assertRaises(
             ValueError,
             self.qs._transform_cand_annot,
@@ -120,6 +153,83 @@ def test__transform_cand_annot(self):
             ),
             enforce_mapping=True,
         )
+
+        X = np.array([[1, 2], [0, 1]])
+        y = np.array([[1, MISSING_LABEL], [2, 3]])
+        candidates_values = [
+            None,
+            np.array([0, 1]),
+            np.array([[3, 4], [0, 1]]),
+        ]
+        annotators_values = [
+            None,
+            np.array([0]),
+            np.array([[False, True], [True, True]]),
+        ]
+
+        for (i, candidates), (j, annotators) in product(
+            enumerate(candidates_values), enumerate(annotators_values)
+        ):
+            X_cand, mapping, A_cand = self.qs._transform_cand_annot(
+                candidates=candidates,
+                annotators=annotators,
+                X=X,
+                y=y,
+            )
+            self.assertEqual(len(A_cand), len(X_cand))
+
+            if i == 0 and j == 0:
+                np.testing.assert_array_equal(A_cand, is_unlabeled(y)[mapping])
+                np.testing.assert_array_equal(X[mapping], X_cand)
+                np.testing.assert_array_equal(
+                    mapping, np.nonzero(np.any(is_unlabeled(y), axis=1))[0]
+                )
+            if i == 0 and j == 1:
+                expected_A_cand = np.full((len(X_cand), len(y.T)), False)
+                expected_A_cand[:, annotators] = True
+                np.testing.assert_array_equal(A_cand, expected_A_cand)
+                np.testing.assert_array_equal(X[mapping], X_cand)
+            if i == 0 and j == 2:
+                np.testing.assert_array_equal(annotators[mapping], A_cand)
+                np.testing.assert_array_equal(X[mapping], X_cand)
+                np.testing.assert_array_equal(
+                    mapping, np.nonzero(np.any(A_cand, axis=1))[0]
+                )
+            if i == 1 and j == 0:
+                np.testing.assert_array_equal(X[mapping], X_cand)
+                np.testing.assert_array_equal(
+                    A_cand, np.full((len(X_cand), len(y.T)), True)
+                )
+                np.testing.assert_array_equal(mapping, candidates)
+            if i == 1 and j == 1:
+                expected_A_cand = np.full((len(X_cand), len(y.T)), False)
+                expected_A_cand[:, annotators] = True
+                np.testing.assert_array_equal(A_cand, expected_A_cand)
+                np.testing.assert_array_equal(X[mapping], X_cand)
+                np.testing.assert_array_equal(mapping, candidates)
+            if i == 1 and j == 2:
+                np.testing.assert_array_equal(annotators[mapping], A_cand)
+                np.testing.assert_array_equal(X[mapping], X_cand)
+                np.testing.assert_array_equal(
+                    mapping, candidates[np.any(A_cand, axis=1)]
+                )
+            if i == 2 and j == 0:
+                self.assertEqual(mapping, None)
+                np.testing.assert_array_equal(X_cand, candidates)
+                np.testing.assert_array_equal(
+                    A_cand, np.full((len(X_cand), len(y.T)), True)
+                )
+            if i == 2 and j == 1:
+                self.assertEqual(mapping, None)
+                np.testing.assert_array_equal(X_cand, candidates)
+                expected_A_cand = np.full((len(X_cand), len(y.T)), False)
+                expected_A_cand[:, annotators] = True
+                np.testing.assert_array_equal(A_cand, expected_A_cand)
+            if i == 2 and j == 2:
+                self.assertEqual(mapping, None)
+                np.testing.assert_array_equal(X_cand, candidates)
+                np.testing.assert_array_equal(A_cand, annotators)
+
         re_val = self.qs._transform_cand_annot(
             candidates=np.arange(2),
             annotators=np.arange(2),
@@ -127,7 +237,7 @@ def test__transform_cand_annot(self):
             y=np.array([[1, MISSING_LABEL], [2, 3]]),
         )
         X_cand, mapping, A_cand = re_val
-        np.testing.assert_array_equal(X_cand, np.array([[1, 2]]))
+        np.testing.assert_array_equal(X_cand, np.array([[1, 2], [0, 1]]))
 
         re_val = self.qs._transform_cand_annot(
             candidates=None,
@@ -140,53 +250,43 @@ def test__transform_cand_annot(self):
             A_cand, np.array([[False, True], [True, True]])
         )
 
-        re_val = self.qs._validate_data(
-            candidates=None,
-            annotators=[1],
-            X=np.array([[1, 2], [0, 1]]),
-            y=np.array([[1, MISSING_LABEL], [2, 3]]),
-            batch_size=2,
-            return_utilities=False,
-        )
-
-        X, y, candidates, annotators, batch_size, return_utilities = re_val
-        self.assertEqual(1, batch_size)
-
-        re_val = self.qs._validate_data(
-            candidates=None,
-            annotators=np.array([[False, True], [True, True]]),
-            X=np.array([[1, 2], [0, 1]]),
-            y=np.array([[1, MISSING_LABEL], [2, 3]]),
-            batch_size=2,
-            return_utilities=False,
-        )
-
-        X, y, candidates, annotators, batch_size, return_utilities = re_val
-        self.assertEqual(2, batch_size)
-
-        re_val = self.qs._validate_data(
-            candidates=np.array([[1, 2], [0, 1]]),
-            annotators=np.array([[False, True], [True, True]]),
-            X=np.array([[1, 2], [0, 1]]),
-            y=np.array([[1, MISSING_LABEL], [2, 3]]),
-            batch_size=2,
-            return_utilities=False,
-        )
-
-        X, y, candidates, annotators, batch_size, return_utilities = re_val
-        self.assertEqual(2, batch_size)
-
-        re_val = self.qs._validate_data(
-            candidates=[1],
-            annotators=[1],
-            X=np.array([[1, 2], [0, 1]]),
-            y=np.array([[1, MISSING_LABEL], [2, 3]]),
-            batch_size=2,
-            return_utilities=False,
-        )
-
-        X, y, candidates, annotators, batch_size, return_utilities = re_val
-        self.assertEqual(0, batch_size)
+    def test_consistency_validate_and_transform(self):
+        X = np.array([[1, 2], [0, 1]])
+        y = np.array([[1, MISSING_LABEL], [2, 3]])
+        batch_size_initial = y.shape[0] * y.shape[1]
+        candidates_values = [
+            None,
+            np.array([0, 1]),
+            np.array([[3, 4], [0, 1]]),
+        ]
+        annotators_values = [
+            None,
+            np.array([0]),
+            np.array([[False, True], [True, True]]),
+        ]
+
+        for (i, candidates), (j, annotators) in product(
+            enumerate(candidates_values), enumerate(annotators_values)
+        ):
+            X, y, candidates, annotators, batch_size, return_utilities = (
+                self.qs._validate_data(
+                    candidates=candidates,
+                    annotators=annotators,
+                    X=X,
+                    y=y,
+                    batch_size=batch_size_initial,
+                    return_utilities=False,
+                )
+            )
+
+            X_cand, mapping, A_cand = self.qs._transform_cand_annot(
+                candidates=candidates,
+                annotators=annotators,
+                X=X,
+                y=y,
+            )
+
+            self.assertEqual(np.sum(A_cand).item(), batch_size)
 
 
 class SkactivemlClassifierTest(unittest.TestCase):