Repository: scikit-activeml/scikit-activeml (language: python)

Issue:
FR: Extend `QueryByCommittee`
### Describe the workflow you want to enable

There are two interesting extensions for `QueryByCommittee`:

A. There are Bayesian models implementing sampling mechanisms, which can be used to construct ensembles.
B. A popular method to measure disagreement is the variation ratio [1], which should be added.

[1] [Beluch, W. H., Genewein, T., Nürnberger, A., and Köhler, J. M. The Power of Ensembles for Active Learning in Image Classification. In Conference on Computer Vision and Pattern Recognition, pages 9368-9377, 2018.](https://openaccess.thecvf.com/content_cvpr_2018/papers/Beluch_The_Power_of_CVPR_2018_paper.pdf)

### Describe your proposed solution

Extension A can be implemented by allowing models to implement the method `sample` or `sample_y.` As an example, the `ParzenWindowClassifier` can be extended to provide such a sampling mechanism via the Dirichlet distribution.

Extension B can be implemented by adding `method='variation_ratios'` as possible option, which computed the variation ratios according to Eq. (4) in [1].

### Describe alternatives you've considered, if relevant

_No response_

### Additional context

_No response_


Discussion / hints:

Fix patch:
diff --git a/docs/examples/pool/_query_by_committee.json b/docs/examples/pool/_query_by_committee.json
index 56befdf2..9d7ade63 100644
--- a/docs/examples/pool/_query_by_committee.json
+++ b/docs/examples/pool/_query_by_committee.json
@@ -2,39 +2,73 @@
   {
     "class" : "QueryByCommittee",
     "package" : "pool",
-    "method" : "Query-by-Committee with Kullback-Leibler Divergence",
+    "method" : "Query-by-Committee (QBC) with Kullback-Leibler Divergence",
     "category" :  "Query-by-Committee",
     "template" : "examples/template_pool.py",
-    "tags" : ["pool",  "classification", "regression", "single-annotator"],
-    "title" : "Query-by-Committee with Kullback-Leibler Divergence",
+    "tags" : ["pool",  "classification", "single-annotator"],
+    "title" : "Query-by-Committee (QBC) with Kullback-Leibler Divergence",
     "refs" : ["seung1992query","mccallum1998employing"],
     "sequence" : ["title", "text_0", "plot", "refs"],
     "text_0" : "",
     "import_misc" : ["from sklearn.ensemble import BaggingClassifier",
                      "from skactiveml.pool import QueryByCommittee"],
-    "import_clf" : "from skactiveml.classifier import SklearnClassifier, ParzenWindowClassifier",
-    "init_clf" : "SklearnClassifier(BaggingClassifier(ParzenWindowClassifier()), classes=np.unique(y_true))",
-    "init_qs" : "QueryByCommittee(method='KL_divergence')",
+    "import_clf" : "from skactiveml.classifier import ParzenWindowClassifier",
+    "init_clf" : "ParzenWindowClassifier(classes=np.unique(y_true), class_prior=0.1)",
+    "init_qs" : "QueryByCommittee(method='KL_divergence', sample_predictions_method_name='sample_proba', sample_predictions_dict={'n_samples': 100})",
     "query_params" : "X=X, y=y, ensemble=clf"
   },
   {
     "class" : "QueryByCommittee",
     "package" : "pool",
-    "method" : "Query-by-Committee with Vote Entropy",
-    "categories" : {
-      "Settles2012" : "Query-by-Committee"
-    },
+    "method" : "Query-by-Committee (QBC) with Vote Entropy",
+    "category" : "Query-by-Committee",
     "template" : "examples/template_pool.py",
     "tags" : ["pool",  "classification", "single-annotator"],
-    "title" : "Query-by-Committee with Vote Entropy",
+    "title" : "Query-by-Committee (QBC) with Vote Entropy",
     "refs" : ["seung1992query","engelson1996minimizing"],
     "sequence" : ["title", "text_0", "plot", "refs"],
     "text_0" : "",
-    "import_misc" : ["from sklearn.ensemble import BaggingClassifier",
-                     "from skactiveml.pool import QueryByCommittee"],
-    "import_clf" : "from skactiveml.classifier import SklearnClassifier, ParzenWindowClassifier",
-    "init_clf" : "SklearnClassifier(BaggingClassifier(ParzenWindowClassifier()), classes=np.unique(y_true))",
-    "init_qs" : "QueryByCommittee(method='vote_entropy')",
+    "import_misc" : ["from skactiveml.pool import QueryByCommittee"],
+    "import_clf" : "from skactiveml.classifier import ParzenWindowClassifier",
+    "init_clf" : "ParzenWindowClassifier(classes=np.unique(y_true), class_prior=0.1)",
+    "init_qs" : "QueryByCommittee(method='vote_entropy', sample_predictions_method_name='sample_proba', sample_predictions_dict={'n_samples': 100})",
+    "query_params" : "X=X, y=y, ensemble=clf"
+  },
+  {
+    "class" : "QueryByCommittee",
+    "package" : "pool",
+    "method" : "Query-by-Committee (QBC) with Variation Ratios",
+    "category" : "Query-by-Committee",
+    "template" : "examples/template_pool.py",
+    "tags" : ["pool",  "classification", "single-annotator"],
+    "title" : "Query-by-Committee (QBC) with Variation Ratios",
+    "refs" : ["seung1992query","beluch2018power"],
+    "sequence" : ["title", "text_0", "plot", "refs"],
+    "text_0" : "",
+    "import_misc" : ["from skactiveml.pool import QueryByCommittee"],
+    "import_clf" : "from skactiveml.classifier import ParzenWindowClassifier",
+    "init_clf" : "ParzenWindowClassifier(classes=np.unique(y_true), class_prior=0.1)",
+    "init_qs" : "QueryByCommittee(method='variation_ratios', sample_predictions_method_name='sample_proba', sample_predictions_dict={'n_samples': 100})",
     "query_params" : "X=X, y=y, ensemble=clf"
+  },
+  {
+    "class" : "QueryByCommittee",
+    "package" : "pool",
+    "method" : "Query-by-Committee (QBC) with Empirical Variance",
+    "category" : "Query-by-Committee",
+    "template" : "examples/template_pool_regression.py",
+    "tags" : ["pool",  "regression", "single-annotator"],
+    "title" : "Query-by-Committee (QBC) with Empirical Variance",
+    "refs" : ["seung1992query","burbidge2007active"],
+    "sequence" : ["title", "text_0", "plot", "refs"],
+    "text_0" : "",
+    "import_misc" : [
+      "from skactiveml.pool import QueryByCommittee",
+      "from skactiveml.regressor import SklearnRegressor"
+    ],
+    "import_reg" : "from sklearn.gaussian_process import GaussianProcessRegressor",
+    "init_reg" : "SklearnRegressor(GaussianProcessRegressor())",
+    "init_qs" : "QueryByCommittee(sample_predictions_method_name='sample_y', sample_predictions_dict={'n_samples': 100})",
+    "query_params" : "X=X, y=y, ensemble=reg"
   }
 ]
diff --git a/docs/examples/template_pool_regression.py b/docs/examples/template_pool_regression.py
index 47036bd8..7448677f 100644
--- a/docs/examples/template_pool_regression.py
+++ b/docs/examples/template_pool_regression.py
@@ -51,7 +51,7 @@ def true_function(X_):
 y = np.full(shape=y_true.shape, fill_value=MISSING_LABEL)
 X_test = np.linspace(0, 2, num="$res|100").reshape(-1, 1)
 
-# Initialise the classifier.
+# Initialise the regressor.
 reg = "$init_reg|NICKernelRegressor(random_state=random_state, metric_dict={'gamma': 15.0})"
 # Initialise the query strategy.
 qs = "$init_qs"
diff --git a/docs/refs.bib b/docs/refs.bib
index beb6cdbf..8f48c59c 100644
--- a/docs/refs.bib
+++ b/docs/refs.bib
@@ -133,13 +133,12 @@ @inproceedings{mccallum1998employing
   pages={359--367},
   year={1998}
 }
-
-@article{engelson1996minimizing,
-  title={Minimizing manual annotation cost in supervised training from corpora},
+@inproceedings{engelson1996minimizing,
+  title={{Minimizing Manual Annotation Cost in Supervised Training from Corpora}},
   author={Engelson, Sean P and Dagan, Ido},
-  journal={arXiv},
-  number={9606030},
-  year={1996}
+  booktitle={Annual Meeting of the Association for Computational Linguistics},
+  year={1996},
+  pages={319--326},
 }
 @inproceedings{kapoor2007selective,
   title={Selective Supervision: Guiding Supervised Learning with Decision-Theoretic Active Learning.},
@@ -299,6 +298,13 @@ @inproceedings{hacohen2022active
   booktitle={International Conference on Machine Learning},
   year={2022},
 }
+@inproceedings{beluch2018power,
+  title={{The Power of Ensembles for Active Learning in Image Classification}},
+  author={Beluch, William H and Genewein, Tim and N{\"u}rnberger, Andreas and K{\"o}hler, Jan M},
+  booktitle={IEEE Conference on Computer Vision and Pattern Recognition},
+  pages={9368--9377},
+  year={2018}
+}
 @inproceedings{yehuda2022active,
   title={{Active Learning Through a Covering Lens}},
   author={Yehuda, Ofer and Dekel, Avihu and Hacohen, Guy and Weinshall, Daphna},
@@ -311,4 +317,11 @@ @inproceedings{margatina2021active
   booktitle={Conference on Empirical Methods in Natural Language Processing},
   pages={650--663},
   year={2021}
+}
+@inproceedings{burbidge2007active,
+  title={Active Learning for Regression Based on Query by Committee},
+  author={Burbidge, Robert and Rowland, Jem J and King, Ross D},
+  booktitle={Internation Conference on Intelligent Data Engineering and Automated Learning},
+  pages={209--218},
+  year={2007},
 }
\ No newline at end of file
diff --git a/skactiveml/base.py b/skactiveml/base.py
index 8d7fab31..bc95902a 100644
--- a/skactiveml/base.py
+++ b/skactiveml/base.py
@@ -1317,6 +1317,47 @@ def predict_proba(self, X):
         P[normalizer == 0, :] = [1 / len(self.classes_)] * len(self.classes_)
         return P
 
+    def sample_proba(self, X, n_samples=10, random_state=None):
+        """Samples probability vectors from Dirichlet distributions whose
+        parameters `alphas` are defined as the sum of the frequency estimates
+        returned by `predict_freq` and the `class_prior`.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_test_samples, n_features)
+            Test samples for which `n_samples` probability vectors are to be
+            sampled.
+        n_samples : int, default=10
+            Number of probability vectors to sample for each `X[i]`.
+        random_state : int or numpy.random.RandomState or None, default=None
+            Ensure reproducibility when sampling probability vectors from the
+            Dirichlet distributions.
+
+        Returns
+        -------
+        P : array-like of shape (n_samples, n_test_samples, n_classes)
+            There are `n_samples` class probability vectors for each test
+            sample in `X`. Classes are ordered according to classes_.
+        """
+        random_state = check_random_state(random_state)
+        alphas = self.predict_freq(X) + self.class_prior_
+        alphas = alphas.repeat(repeats=n_samples, axis=0)
+        if (alphas == 0).any():
+            raise ValueError(
+                "There are zero frequency observations. "
+                "Set `class_prior > 0` to avoid this error."
+            )
+        R = random_state.standard_gamma(alphas)
+        R_sums = R.sum(axis=-1)
+        is_zero = (R_sums == 0.0).ravel()
+        sampled_class_indices = random_state.choice(
+            np.array(R.shape[-1]), size=is_zero.sum()
+        )
+        R[is_zero, sampled_class_indices] = 1.0
+        P = R / R.sum(axis=-1, keepdims=True)
+        P = P.reshape(n_samples, len(X), P.shape[-1], order="F")
+        return P
+
     def _validate_data(
         self,
         X,
diff --git a/skactiveml/classifier/_parzen_window_classifier.py b/skactiveml/classifier/_parzen_window_classifier.py
index 6205b303..718327be 100644
--- a/skactiveml/classifier/_parzen_window_classifier.py
+++ b/skactiveml/classifier/_parzen_window_classifier.py
@@ -8,7 +8,10 @@
 import warnings
 from sklearn.metrics.pairwise import pairwise_kernels, KERNEL_PARAMS
 from sklearn.utils import check_array
-from sklearn.utils.validation import check_is_fitted, check_scalar
+from sklearn.utils.validation import (
+    check_is_fitted,
+    check_scalar,
+)
 
 from ..base import ClassFrequencyEstimator
 from ..utils import MISSING_LABEL, compute_vote_vectors, is_labeled
@@ -28,7 +31,7 @@ class ParzenWindowClassifier(ClassFrequencyEstimator):
         during the fit.
     missing_label : scalar or string or np.nan or None, default=np.nan
         Value to represent a missing label.
-    cost_matrix : array-like of shape (n_classes, n_classes)
+    cost_matrix : array-like of shape (n_classes, n_classes), default=None
         Cost matrix with `cost_matrix[i,j]` indicating cost of predicting class
         `classes[j]` for a sample of class `classes[i]`. Can be only set, if
         `classes` is not none.
@@ -39,16 +42,19 @@ class ParzenWindowClassifier(ClassFrequencyEstimator):
         `class_prior` is a float, `class_prior` indicates the non-negative
         prior number of samples per class.
     metric : str or callable, default='rbf'
-        The metric must a be a valid kernel defined by the function
+        The metric must be a valid kernel defined by the function
         `sklearn.metrics.pairwise.pairwise_kernels`.
     n_neighbors : int or None, default=None
         Number of nearest neighbours. Default is None, which means all
         available samples are considered.
-    metric_dict : dict,
+    metric_dict : dict, default=None
         Any further parameters are passed directly to the kernel function.
-        For the the kernel 'rbf' we allow the use of mean kernel [2] and use
+        For the kernel 'rbf' we allow the use of mean kernel [2] and use
         it when gamma is set to 'mean' (i.e., {'gamma': 'mean'}). While N is
         defined as the labeled data the variance is calculated over all X.
+    random_state : int or RandomState instance or None, default=None
+        Determines random number for 'predict' method. Pass an int for
+        reproducible results across multiple method calls.
 
     Attributes
     ----------
@@ -73,7 +79,7 @@ class `classes_[j]` for a sample of class `classes_[i]`.
     .. [1] O. Chapelle, "Active Learning for Parzen Window Classifier",
        Proceedings of the Tenth International Workshop Artificial Intelligence
        and Statistics, 2005.
-       [2] Chaudhuri, A., Kakde, D., Sadek, C., Gonzalez, L., & Kong, S.,
+    .. [2] Chaudhuri, A., Kakde, D., Sadek, C., Gonzalez, L., & Kong, S.,
        "The Mean and Median Criteria for Kernel Bandwidth Selection for Support
        Vector Data Description" IEEE International Conference on Data
        Mining Workshops (ICDMW), 2017.
@@ -113,9 +119,9 @@ def fit(self, X, y, sample_weight=None):
             samples.
         y : array-like of shape (n_samples)
             It contains the class labels of the training samples.
-        sample_weight : array-like of shape (n_samples)
+        sample_weight : array-like of shape (n_samples), default=None
             It contains the weights of the training samples' class labels.
-            It must have the same shape as y.
+            It must have the same shape as `y`.
 
         Returns
         -------
@@ -186,8 +192,8 @@ def predict_freq(self, X):
 
         Parameters
         ----------
-        X: array-like or shape (n_samples, n_features) or shape
-        (n_samples, m_samples) if metric == 'precomputed'
+        X: array-like or shape (n_samples, n_features) or shape \
+                (n_samples, m_samples) if metric == 'precomputed'
             Input samples.
 
         Returns
@@ -230,6 +236,7 @@ def predict_freq(self, X):
                 F[i, :] = K[i, indices[i]] @ self.V_[indices[i], :]
         return F
 
+    @staticmethod
     def _calculate_mean_gamma(
         N, variance, n_features, delta=(np.sqrt(2) * 1e-6)
     ):
diff --git a/skactiveml/pool/__init__.py b/skactiveml/pool/__init__.py
index 76858925..41303cc6 100644
--- a/skactiveml/pool/__init__.py
+++ b/skactiveml/pool/__init__.py
@@ -23,6 +23,7 @@
     QueryByCommittee,
     average_kl_divergence,
     vote_entropy,
+    variation_ratios,
 )
 from ._quire import Quire
 from ._random_sampling import RandomSampling
@@ -55,6 +56,7 @@
     "Quire",
     "average_kl_divergence",
     "vote_entropy",
+    "variation_ratios",
     "FourDs",
     "CostEmbeddingAL",
     "ExpectedModelChangeMaximization",
diff --git a/skactiveml/pool/_badge.py b/skactiveml/pool/_badge.py
index cb75c474..bf9c34ca 100644
--- a/skactiveml/pool/_badge.py
+++ b/skactiveml/pool/_badge.py
@@ -41,8 +41,8 @@ class Badge(SingleAnnotatorPoolQueryStrategy):
 
     References
     ----------
-    .. [1] J. Ash, Jordan T., Chicheng Zhang, Akshay Krishnamurthy,
-       John Langford, and Alekh Agarwal, "Deep Batch Active Learning by Diverse,
+    .. [1] J. Ash, Jordan T., Chicheng Zhang, Akshay Krishnamurthy, John
+       Langford, and Alekh Agarwal, "Deep Batch Active Learning by Diverse,
        Uncertain Gradient Lower Bounds." ICLR, 2019.
     """
 
diff --git a/skactiveml/pool/_bald.py b/skactiveml/pool/_bald.py
index ad55209c..a61d2730 100644
--- a/skactiveml/pool/_bald.py
+++ b/skactiveml/pool/_bald.py
@@ -1,6 +1,10 @@
 """
 Code is based on https://blackhc.github.io/batchbald_redux/ distributed under
-the Apache-2.0 license.
+the Apache-2.0 license and the associated query strategy is presented in [1].
+
+[1] Kirsch, Andreas, Joost Van Amersfoort, and Yarin Gal. "BatchBALD:
+Efficient and Diverse Batch Acquisition for Deep Bayesian Active
+Learning." Advances in Neural Information Processing Systems 32 (2019).
 """
 
 import numpy as np
@@ -31,28 +35,45 @@ class _GeneralBALD(QueryByCommittee):
 
     Parameters
     ----------
-    n_MC_samples : int > 0, optional (default=n_estimators)
+    n_MC_samples : int > 0, default=n_estimators
         The number of monte carlo samples used for label estimation.
-    greedy_selection : bool, optional (default=False)
+    greedy_selection : bool, default=False
         Flag to either use BatchBALD (`greedy_selection=False`) or a greedy
         (top-k) selection (`greedy_selection=True`) if `batch_size>1`.
-    eps : float  > 0, optional (default=1e-7)
+    eps : float > 0, default=1e-7
         Minimum probability threshold to compute log-probabilities.
-    missing_label : scalar or string or np.nan or None, optional
-        (default=np.nan)
+    sample_predictions_method_name : str, default=None
+        Certain estimators may offer methods enabling to construct a committee
+        by sampling predictions of committee members. This parameter is to
+        indicate the name of such a method.
+            - If `sample_predictions_method_name=None` no sampling is
+              performed.
+            - If `sample_predictions_method_name` is not `None` and in the
+              case of classification, the method is expected to take samples of
+              the shape `(n_samples, *)` as input and to output probabilities
+              of the shape `(n_members, n_samples, n_classes)`, e.g.,
+              `sample_proba` in `skactiveml.base.ClassFrequencyEstimator`.
+    sample_predictions_dict : dict, default=None
+        Parameters (excluding the samples) that are passed to the method with
+        the name `sample_predictions_method_name`.
+            - This parameter must be `None`, if
+              `sample_predictions_method_name` is `None`.
+            - Otherwise, it may be used to define the number of sampled
+              members, e.g., by defining `n_samples` as parameter to the method
+              `sample_proba` of `skactiveml.base.ClassFrequencyEstimator`.
+    missing_label : scalar or string or np.nan or None, default=np.nan
         Value to represent a missing label.
-    random_state : int or None or np.random.RandomState, optional
-        (default=None)
+    random_state : int or None or np.random.RandomState, default=None
         The random state to use.
 
     References
     ----------
-    [1] Houlsby, Neil, Ferenc Huszár, Zoubin Ghahramani, and Máté Lengyel.
-        "Bayesian Active Learning for Classification and Preference Learning."
-        arXiv preprint arXiv:1112.5745 (2011).
-    [2] Kirsch, Andreas, Joost Van Amersfoort, and Yarin Gal. "BatchBALD:
-        Efficient and Diverse Batch Acquisition for Deep Bayesian  Active
-        Learning." Advances in Neural Information Processing Systems 32 (2019).
+    .. [1] Houlsby, Neil, Ferenc Huszár, Zoubin Ghahramani, and Máté Lengyel.
+       "Bayesian Active Learning for Classification and Preference
+       Learning." arXiv preprint arXiv:1112.5745 (2011).
+    .. [2] Kirsch, Andreas, Joost Van Amersfoort, and Yarin Gal. "BatchBALD:
+       Efficient and Diverse Batch Acquisition for Deep Bayesian Active
+       Learning." Advances in Neural Information Processing Systems 32 (2019).
     """
 
     def __init__(
@@ -60,15 +81,20 @@ def __init__(
         n_MC_samples=None,
         greedy_selection=False,
         eps=1e-7,
+        sample_predictions_method_name=None,
+        sample_predictions_dict=None,
         missing_label=MISSING_LABEL,
         random_state=None,
     ):
         super().__init__(
-            missing_label=missing_label, random_state=random_state
+            eps=eps,
+            sample_predictions_method_name=sample_predictions_method_name,
+            sample_predictions_dict=sample_predictions_dict,
+            missing_label=missing_label,
+            random_state=random_state,
         )
         self.n_MC_samples = n_MC_samples
         self.greedy_selection = greedy_selection
-        self.eps = eps
 
     def query(
         self,
@@ -88,35 +114,39 @@ def query(
         X : array-like of shape (n_samples, n_features)
             Training data set, usually complete, i.e. including the labeled and
             unlabeled samples.
-        y : array-like of shape (n_samples)
+        y : array-like of shape (n_samples,)
             Labels of the training data set (possibly including unlabeled ones
-            indicated by self.MISSING_LABEL.)
-        ensemble : list or tuple of SkactivemlClassifier or
-            SkactivemlClassifier.
-            If `ensemble` is a `SkactivemlClassifier`, it must have
-            `n_estimators` and `estimators_` after fitting as
-            attribute. Then, its estimators will be used as committee. If
-            `ensemble` is array-like, each element of this list must be
-            `SkactivemlClassifier` or a `SkactivemlRegressor` and will be used
-            as committee member.
+            indicated by `self.missing_label`).
+        ensemble : array-like of SkactivemlClassifier or SkactivemlClassifier
+            - If `ensemble` is a `SkactivemlClassifier` and has
+              `n_estimators` plus `estimators_` after fitting as
+              attributes, its estimators will be used as committee.
+            - If `ensemble` is array-like, each element of this list must be
+              `SkactivemlClassifier` and will be
+              used as committee member.
+            - If `ensemble` is a `SkactivemlClassifier` and implements a
+              method with the name `sample_predictions_method_name`, this
+              method is used to sample predictions of committee members.
         fit_ensemble : bool, default=True
             Defines whether the ensemble should be fitted on `X`, `y`, and
             `sample_weight`.
         sample_weight: array-like of shape (n_samples), default=None
             Weights of training samples in `X`.
-        candidates : None or array-like of shape (n_candidates), dtype=int or
+        candidates : None or array-like of shape (n_candidates), dtype=int or \
                 array-like of shape (n_candidates, n_features), default=None
-            If candidates is None, the unlabeled samples from (X,y) are
-            considered as candidates.
-            If candidates is of shape (n_candidates) and of type int,
-            candidates is considered as the indices of the samples in (X,y).
-            If candidates is of shape (n_candidates, n_features), the
-            candidates are directly given in candidates (not necessarily
-            contained in X). This is not supported by all query strategies.
+            - If `candidates` is `None`, the unlabeled samples from
+              `(X,y)` are considered as `candidates`.
+            - If `candidates` is of shape `(n_candidates,)` and of type
+              `int`, `candidates` is considered as the indices of the
+              samples in `(X,y)`.
+            - If `candidates` is of shape `(n_candidates, *)`, the
+              candidate samples are directly given in `candidates` (not
+              necessarily contained in `X`). This is not supported by all
+              query strategies.
         batch_size : int, default=1
             The number of samples to be selected in one AL cycle.
         return_utilities : bool, default=False
-            If true, also return the utilities based on the query strategy.
+            If `True`, also return the utilities based on the query strategy.
 
         Returns
         -------
@@ -124,20 +154,22 @@ def query(
             The query_indices indicate for which candidate sample a label is
             to queried, e.g., `query_indices[0]` indicates the first selected
             sample.
-            If candidates is None or of shape (n_candidates), the indexing
-            refers to samples in X.
-            If candidates is of shape (n_candidates, n_features), the indexing
-            refers to samples in candidates.
-        utilities : numpy.ndarray of shape (batch_size, n_samples) or
+                - If `candidates` is `None` or of shape
+                  `(n_candidates,)`, the indexing refers to the samples in
+                  `X`.
+                - If `candidates` is of shape `(n_candidates, n_features)`,
+                  the indexing refers to the samples in `candidates`.
+        utilities : numpy.ndarray of shape (batch_size, n_samples) or \
                 numpy.ndarray of shape (batch_size, n_candidates)
             The utilities of samples after each selected sample of the batch,
             e.g., `utilities[0]` indicates the utilities used for selecting
             the first sample (with index `query_indices[0]`) of the batch.
             Utilities for labeled samples will be set to np.nan.
-            If candidates is None or of shape (n_candidates), the indexing
-            refers to samples in X.
-            If candidates is of shape (n_candidates, n_features), the indexing
-            refers to samples in candidates.
+                - If `candidates` is `None` or of shape
+                  `(n_candidates,)`, the indexing refers to the samples in
+                  `X`.
+                - If `candidates` is of shape `(n_candidates, n_features)`,
+                  the indexing refers to the samples in `candidates`.
         """
         # Validate input parameters.
         X, y, candidates, batch_size, return_utilities = self._validate_data(
@@ -151,7 +183,7 @@ def query(
         # Validate classifier type.
         check_type(fit_ensemble, "fit_ensemble", bool)
 
-        ensemble, est_arr, _ = _check_ensemble(
+        ensemble, est_arr, _, sample_func, sample_dict = _check_ensemble(
             ensemble=ensemble,
             X=X,
             y=y,
@@ -159,14 +191,17 @@ def query(
             fit_ensemble=fit_ensemble,
             missing_label=self.missing_label_,
             estimator_types=[SkactivemlClassifier],
+            sample_predictions_method_name=self.sample_predictions_method_name,
+            sample_predictions_dict=self.sample_predictions_dict,
         )
 
-        probas = self._aggregate_predict_probas(X_cand, ensemble, est_arr)
+        if sample_func is None:
+            probas = self._aggregate_predict_probas(X_cand, ensemble, est_arr)
+        else:
+            probas = sample_func(X_cand, **sample_dict)
 
-        # probas = np.array([est.predict_proba(X_cand) for est in est_arr])
-        # print(probas[:,:5,:])
         if self.n_MC_samples is None:
-            n_MC_samples_ = len(est_arr)
+            n_MC_samples_ = len(probas)
         else:
             n_MC_samples_ = self.n_MC_samples
         check_scalar(n_MC_samples_, "n_MC_samples", int, min_val=1)
@@ -206,39 +241,54 @@ def query(
 class BatchBALD(_GeneralBALD):
     """Batch Bayesian Active Learning by Disagreement (BatchBALD)
 
-    The Bayesian Active Learning by Disagreement (BatchBALD) [1] strategy
+    The Batch Bayesian Active Learning by Disagreement (BatchBALD) [1] strategy
     reduces the number of possible hypotheses maximally fast to minimize the
     uncertainty about the parameters using Shannon's entropy. It seeks the data
-    point that maximises the decrease in expected posterior entropy. For the
-    batch case, the advanced strategy BatchBALD [2] is used.
+    point that maximises the decrease in expected posterior entropy.
 
     Parameters
     ----------
-    n_MC_samples : int > 0, optional (default=n_estimators)
+    n_MC_samples : int > 0, default=n_estimators
         The number of monte carlo samples used for label estimation.
-    eps : float  > 0, optional (default=1e-7)
+    eps : float > 0, default=1e-7
         Minimum probability threshold to compute log-probabilities.
-    missing_label : scalar or string or np.nan or None, optional
-        (default=np.nan)
+    sample_predictions_method_name : str, default=None
+        Certain estimators may offer methods enabling to construct a committee
+        by sampling predictions of committee members. This parameter is to
+        indicate the name of such a method.
+            - If `sample_predictions_method_name=None` no sampling is
+              performed.
+            - If `sample_predictions_method_name` is not `None` and in the
+              case of classification, the method is expected to take samples of
+              the shape `(n_samples, *)` as input and to output probabilities
+              of the shape `(n_members, n_samples, n_classes)`, e.g.,
+              `sample_proba` in `skactiveml.base.ClassFrequencyEstimator`.
+    sample_predictions_dict : dict, default=None
+        Parameters (excluding the samples) that are passed to the method with
+        the name `sample_predictions_method_name`.
+            - This parameter must be `None`, if
+              `sample_predictions_method_name` is `None`.
+            - Otherwise, it may be used to define the number of sampled
+              members, e.g., by defining `n_samples` as parameter to the method
+              `sample_proba` of `skactiveml.base.ClassFrequencyEstimator`.
+    missing_label : scalar or string or np.nan or None, default=np.nan
         Value to represent a missing label.
-    random_state : int or None or np.random.RandomState, optional
-        (default=None)
+    random_state : int or None or np.random.RandomState, default=None
         The random state to use.
 
     References
     ----------
-    [1] Houlsby, Neil, Ferenc Huszár, Zoubin Ghahramani, and Máté Lengyel.
-        "Bayesian Active Learning for Classification and Preference Learning."
-        arXiv preprint arXiv:1112.5745 (2011).
-    [2] Kirsch, Andreas, Joost Van Amersfoort, and Yarin Gal. "BatchBALD:
-        Efficient and Diverse Batch Acquisition for Deep Bayesian  Active
-        Learning." Advances in Neural Information Processing Systems 32 (2019).
+    .. [1] Kirsch, Andreas, Joost Van Amersfoort, and Yarin Gal. "BatchBALD:
+       Efficient and Diverse Batch Acquisition for Deep Bayesian  Active
+       Learning." Advances in Neural Information Processing Systems 32 (2019).
     """
 
     def __init__(
         self,
         n_MC_samples=None,
         eps=1e-7,
+        sample_predictions_method_name=None,
+        sample_predictions_dict=None,
         missing_label=MISSING_LABEL,
         random_state=None,
     ):
@@ -246,6 +296,8 @@ def __init__(
             n_MC_samples=n_MC_samples,
             greedy_selection=False,
             eps=eps,
+            sample_predictions_method_name=sample_predictions_method_name,
+            sample_predictions_dict=sample_predictions_dict,
             missing_label=missing_label,
             random_state=random_state,
         )
@@ -262,28 +314,47 @@ class GreedyBALD(_GeneralBALD):
 
     Parameters
     ----------
-    n_MC_samples : int > 0, optional (default=n_estimators)
+    n_MC_samples : int > 0, default=n_estimators
         The number of monte carlo samples used for label estimation.
-    eps : float  > 0, optional (default=1e-7)
+    eps : float > 0, default=1e-7
         Minimum probability threshold to compute log-probabilities.
-    missing_label : scalar or string or np.nan or None, optional
-        (default=np.nan)
+    sample_predictions_method_name : str, default=None
+        Certain estimators may offer methods enabling to construct a committee
+        by sampling predictions of committee members. This parameter is to
+        indicate the name of such a method.
+            - If `sample_predictions_method_name=None` no sampling is
+              performed.
+            - If `sample_predictions_method_name` is not `None` and in the
+              case of classification, the method is expected to take samples of
+              the shape `(n_samples, *)` as input and to output probabilities
+              of the shape `(n_members, n_samples, n_classes)`, e.g.,
+              `sample_proba` in `skactiveml.base.ClassFrequencyEstimator`.
+    sample_predictions_dict : dict, default=None
+        Parameters (excluding the samples) that are passed to the method with
+        the name `sample_predictions_method_name`.
+            - This parameter must be `None`, if
+              `sample_predictions_method_name` is `None`.
+            - Otherwise, it may be used to define the number of sampled
+              members, e.g., by defining `n_samples` as parameter to the method
+              `sample_proba` of `skactiveml.base.ClassFrequencyEstimator`.
+    missing_label : scalar or string or np.nan or None, default=np.nan
         Value to represent a missing label.
-    random_state : int or None or np.random.RandomState, optional
-        (default=None)
+    random_state : int or None or np.random.RandomState, default=None
         The random state to use.
 
     References
     ----------
-    [1] Houlsby, Neil, Ferenc Huszár, Zoubin Ghahramani, and Máté Lengyel.
-        "Bayesian Active Learning for Classification and Preference Learning."
-        arXiv preprint arXiv:1112.5745 (2011).
+    .. [1] Houlsby, Neil, Ferenc Huszár, Zoubin Ghahramani, and Máté Lengyel.
+       "Bayesian Active Learning for Classification and Preference Learning."
+       arXiv preprint arXiv:1112.5745 (2011).
     """
 
     def __init__(
         self,
         n_MC_samples=None,
         eps=1e-7,
+        sample_predictions_method_name=None,
+        sample_predictions_dict=None,
         missing_label=MISSING_LABEL,
         random_state=None,
     ):
@@ -291,6 +362,8 @@ def __init__(
             n_MC_samples=n_MC_samples,
             greedy_selection=True,
             eps=eps,
+            sample_predictions_method_name=sample_predictions_method_name,
+            sample_predictions_dict=sample_predictions_dict,
             missing_label=missing_label,
             random_state=random_state,
         )
@@ -306,7 +379,7 @@ def batch_bald(
     """BatchBALD: Efficient and Diverse Batch Acquisition for Deep Bayesian
     Active Learning
 
-    BatchBALD [2] is an extension of BALD  [1] (Bayesian Active Learning by
+    BatchBALD [1] is an extension of BALD  [2] (Bayesian Active Learning by
     Disagreement) whereby points are jointly scored by estimating the
     mutual information between a joint of multiple data points and the model
     parameters.
@@ -315,11 +388,11 @@ def batch_bald(
     ----------
     probas : array-like of shape (n_estimators, n_samples, n_classes)
         The probability estimates of all estimators, samples, and classes.
-    batch_size : int, optional (default=1)
+    batch_size : int, default=1
         The number of samples to be selected in one AL cycle.
-    n_MC_samples : int > 0, optional (default=n_estimators)
+    n_MC_samples : int > 0, default=n_estimators
         The number of monte carlo samples used for label estimation.
-    eps : float  > 0, optional (default=1e-7)
+    eps : float  > 0, default=1e-7
         Minimum probability threshold to compute log-probabilities.
     random_state : int or np.random.RandomState, default=None
         The random state to use.
@@ -331,12 +404,12 @@ def batch_bald(
 
     References
     ----------
-    [1] Houlsby, Neil, Ferenc Huszár, Zoubin Ghahramani, and Máté Lengyel.
-        "Bayesian Active Learning for Classification and Preference Learning."
-        arXiv preprint arXiv:1112.5745 (2011).
-    [2] Kirsch, Andreas, Joost Van Amersfoort, and Yarin Gal. "BatchBALD:
-        Efficient and Diverse Batch Acquisition for Deep Bayesian  Active
-        Learning." Advances in Neural Information Processing Systems 32 (2019).
+    .. [1] Kirsch, Andreas, Joost Van Amersfoort, and Yarin Gal. "BatchBALD:
+       Efficient and Diverse Batch Acquisition for Deep Bayesian  Active
+       Learning." Advances in Neural Information Processing Systems 32 (2019).
+    .. [2] Houlsby, Neil, Ferenc Huszár, Zoubin Ghahramani, and Máté Lengyel.
+       "Bayesian Active Learning for Classification and Preference Learning."
+       arXiv preprint arXiv:1112.5745 (2011).
     """
     # Validate input parameters.
     if probas.ndim != 3:
diff --git a/skactiveml/pool/_contrastive_al.py b/skactiveml/pool/_contrastive_al.py
index ba1ad9c1..025727b5 100644
--- a/skactiveml/pool/_contrastive_al.py
+++ b/skactiveml/pool/_contrastive_al.py
@@ -1,6 +1,6 @@
 """
-Module implementing `ContrastiveAL`, which is a deep active learning strategy selecting
-contrastive samples.
+Module implementing `ContrastiveAL`, which is a deep active learning strategy
+selecting contrastive samples.
 """
 
 import numpy as np
@@ -22,10 +22,10 @@
 class ContrastiveAL(SingleAnnotatorPoolQueryStrategy):
     """Contrastive Active Learning (ContrastiveAL)
 
-    This class implements the Contrastive Active Learning (ContrastiveAL) query strategy
-    [1], which  selects samples similar in the (classifier's learned) feature
-    space, while the classifier predicts maximally different class-membership
-    probabilities.
+    This class implements the Contrastive Active Learning (ContrastiveAL) query
+    strategy [1], which  selects samples similar in the (classifier's learned)
+    feature space, while the classifier predicts maximally different
+    class-membership probabilities.
 
     Parameters
     ----------
diff --git a/skactiveml/pool/_cost_embedding_al.py b/skactiveml/pool/_cost_embedding_al.py
index ace145a1..61ae6b13 100644
--- a/skactiveml/pool/_cost_embedding_al.py
+++ b/skactiveml/pool/_cost_embedding_al.py
@@ -741,10 +741,10 @@ def fit_transform(self, X, y=None, init=None):
         X = check_array(X)
         if X.shape[0] == X.shape[1] and self.dissimilarity != "precomputed":
             warnings.warn(
-                "The MDS API has changed. ``fit`` now constructs an"
+                "The MDS API has changed. `fit` now constructs an"
                 " dissimilarity matrix from data. To use a custom "
                 "dissimilarity matrix, set "
-                "``dissimilarity=precomputed``."
+                "`dissimilarity=precomputed`."
             )
 
         if self.dissimilarity == "precomputed":
diff --git a/skactiveml/pool/_query_by_committee.py b/skactiveml/pool/_query_by_committee.py
index b661a115..4a8e23fb 100644
--- a/skactiveml/pool/_query_by_committee.py
+++ b/skactiveml/pool/_query_by_committee.py
@@ -27,43 +27,79 @@
 
 
 class QueryByCommittee(SingleAnnotatorPoolQueryStrategy):
-    """Query-by-Committee.
+    """Query-by-Committee (QBC)
 
-    The Query-by-Committee (QueryByCommittee) strategy uses an ensemble of
-    estimators to identify on which instances many estimators disagree.
+    The Query-by-Committee (QBC) strategy uses an ensemble of estimators to
+    identify on which samples many estimators disagree.
 
     Parameters
     ----------
-    method : string, default='KL_divergence'
+    method : "KL_divergence" or "vote_entropy" or "variation_ratios, \
+            default='KL_divergence'
         The method to calculate the disagreement in the case of classification.
-        KL_divergence or vote_entropy are possible. In the case of regression
-        the empirical variance is used.
-    eps : float  > 0, optional (default=1e-7)
+        'KL_divergence', 'vote_entropy', and 'variation_ratios' are possible.
+        In the case of regression, this parameter is ignored and the empirical
+        variance is used.
+    eps : float > 0, default=1e-7
         Minimum probability threshold to compute log-probabilities (only
         relevant for `method='KL_divergence'`).
+    sample_predictions_method_name : str, default=None
+        Certain estimators may offer methods enabling to construct a committee
+        by sampling predictions of committee members. This parameter is to
+        indicate the name of such a method.
+            - If `sample_predictions_method_name=None` no sampling is
+              performed.
+            - If `sample_predictions_method_name` is not `None` and in the
+              case of classification, the method is expected to take samples of
+              the shape `(n_samples, *)` as input and to output probabilities
+              of the shape `(n_members, n_samples, n_classes)`, e.g.,
+              `sample_proba` in `skactiveml.base.ClassFrequencyEstimator`.
+            - If `sample_predictions_method_name` is not `None` and in the
+              case of regression, the method is expected to take samples of the
+              shape `(n_samples, *)` as input and to output numerical values of
+              the shape `(n_members, n_samples)`, e.g., `sample_y` in
+              `sklearn.gaussian_process.GaussianProcessRegressor`.
+    sample_predictions_dict : dict, default=None
+        Parameters (excluding the samples) that are passed to the method with
+        the name `sample_predictions_method_name`.
+            - This parameter must be `None`, if
+              `sample_predictions_method_name` is `None`.
+            - Otherwise, it may be used to define the number of sampled
+              members, e.g., by defining `n_samples` as parameter to the method
+              `sample_proba` of `skactiveml.base.ClassFrequencyEstimator` or
+              `sample_y` of
+              `sklearn.gaussian_process.GaussianProcessRegressor`.
     missing_label : scalar or string or np.nan or None, default=np.nan
         Value to represent a missing label.
-    random_state : int or np.random.RandomState, default=None
+    random_state : int or np.random.RandomState or None, default=None
         The random state to use.
 
     References
     ----------
-    [1] H.S. Seung, M. Opper, and H. Sompolinsky. Query by committee.
-        In Proceedings of the ACM Workshop on Computational Learning Theory,
-        pages 287-294, 1992.
-    [2] N. Abe and H. Mamitsuka. Query learning strategies using boosting and
-        bagging. In Proceedings of the International Conference on Machine
-        Learning (ICML), pages 1-9. Morgan Kaufmann, 1998.
-    [3] Burbidge, Robert and Rowland, Jem J and King, Ross D. Active learning
-        for regression based on query by committee. International conference on
-        intelligent data engineering and automated learning, pages 209--218,
-        2007.
+    .. [1] H.S. Seung, M. Opper, and H. Sompolinsky. Query by committee.
+       In ACM Workshop on Computational Learning Theory, pages 287-294, 1992.
+
+    .. [2] N. Abe and H. Mamitsuka. Query Learning Strategies Using Boosting
+       and Bagging. In International Conference on Machine Learning, pages 1-9,
+       1998.
+
+    .. [3] Burbidge, Robert and Rowland, Jem J and King, Ross D. Active
+       Learning for Regression Based on Query by Committee. In International
+       Conference on Intelligent Data Engineering and Automated Learning,
+       pages 209-218, 2007.
+
+    .. [4] Beluch, W. H., Genewein, T., Nürnberger, A., and Köhler, J. M.
+       The Power of Ensembles for Active Learning in Image Classification. In
+       Conference on Computer Vision and Pattern Recognition, pages 9368-9377,
+       2018
     """
 
     def __init__(
         self,
         method="KL_divergence",
         eps=1e-7,
+        sample_predictions_method_name=None,
+        sample_predictions_dict=None,
         missing_label=MISSING_LABEL,
         random_state=None,
     ):
@@ -72,6 +108,8 @@ def __init__(
         )
         self.method = method
         self.eps = eps
+        self.sample_predictions_method_name = sample_predictions_method_name
+        self.sample_predictions_dict = sample_predictions_dict
 
     def query(
         self,
@@ -91,31 +129,39 @@ def query(
         X : array-like of shape (n_samples, n_features)
             Training data set, usually complete, i.e. including the labeled and
             unlabeled samples.
-        y : array-like of shape (n_samples)
+        y : array-like of shape (n_samples,)
             Labels of the training data set (possibly including unlabeled ones
-            indicated by self.MISSING_LABEL.)
-        ensemble : list or tuple of SkactivemlClassifier or list or tuple of
-        SkactivemlRegressor, SkactivemlClassifier or SkactivemlRegressor
-            If `ensemble` is a `SkactivemlClassifier` or a
-            `SkactivemlRegressor`, it must have `n_estimators` and
-            `estimators_` after fitting as attribute. Then, its estimators will
-            be used as committee. If `ensemble` is array-like, each element of
-            this list must be `SkactivemlClassifier` or a `SkactivemlRegressor`
-            and will be used as committee member.
+            indicated by `self.missing_label`.)
+        ensemble : array-like of SkactivemlClassifier or array-like of \
+                SkactivemlRegressor or SkactivemlClassifier or \
+                SkactivemlRegressor
+            - If `ensemble` is a `SkactivemlClassifier` or a
+              `SkactivemlRegressor` and has `n_estimators` plus
+              `estimators_` after fitting as attributes, its estimators will
+              be used as committee.
+            - If `ensemble` is array-like, each element of this list must be
+              `SkactivemlClassifier` or a `SkactivemlRegressor` and will be
+              used as committee member.
+            - If `ensemble` is a `SkactivemlClassifier` or a
+              `SkactivemlRegressor` and implements a method with the name
+              `sample_predictions_method_name`, this method is used to sample
+              predictions of committee members.
         fit_ensemble : bool, default=True
             Defines whether the ensemble should be fitted on `X`, `y`, and
             `sample_weight`.
-        sample_weight: array-like of shape (n_samples), default=None
+        sample_weight: array-like of shape (n_samples,), default=None
             Weights of training samples in `X`.
-        candidates : None or array-like of shape (n_candidates), dtype=int or
+        candidates : None or array-like of shape (n_candidates), dtype=int or \
                 array-like of shape (n_candidates, n_features), default=None
-            If candidates is None, the unlabeled samples from (X,y) are
-            considered as candidates.
-            If candidates is of shape (n_candidates) and of type int,
-            candidates is considered as the indices of the samples in (X,y).
-            If candidates is of shape (n_candidates, n_features), the
-            candidates are directly given in candidates (not necessarily
-            contained in X). This is not supported by all query strategies.
+            - If `candidates` is `None`, the unlabeled samples from
+              `(X,y)` are considered as `candidates`.
+            - If `candidates` is of shape `(n_candidates,)` and of type
+              `int`, `candidates` is considered as the indices of the
+              samples in `(X,y)`.
+            - If `candidates` is of shape `(n_candidates, *)`, the
+              candidate samples are directly given in `candidates` (not
+              necessarily contained in `X`). This is not supported by all
+              query strategies.
         batch_size : int, default=1
             The number of samples to be selected in one AL cycle.
         return_utilities : bool, default=False
@@ -127,20 +173,22 @@ def query(
             The query_indices indicate for which candidate sample a label is
             to queried, e.g., `query_indices[0]` indicates the first selected
             sample.
-            If candidates is None or of shape (n_candidates), the indexing
-            refers to samples in X.
-            If candidates is of shape (n_candidates, n_features), the indexing
-            refers to samples in candidates.
-        utilities : numpy.ndarray of shape (batch_size, n_samples) or
+                - If `candidates` is `None` or of shape
+                  `(n_candidates,)`, the indexing refers to the samples in
+                  `X`.
+                - If `candidates` is of shape `(n_candidates, n_features)`,
+                  the indexing refers to the samples in `candidates`.
+        utilities : numpy.ndarray of shape (batch_size, n_samples) or \
                 numpy.ndarray of shape (batch_size, n_candidates)
             The utilities of samples after each selected sample of the batch,
             e.g., `utilities[0]` indicates the utilities used for selecting
             the first sample (with index `query_indices[0]`) of the batch.
             Utilities for labeled samples will be set to np.nan.
-            If candidates is None or of shape (n_candidates), the indexing
-            refers to samples in X.
-            If candidates is of shape (n_candidates, n_features), the indexing
-            refers to samples in candidates.
+                - If `candidates` is `None` or of shape
+                  `(n_candidates,)`, the indexing refers to the samples in
+                  `X`.
+                - If `candidates` is of shape `(n_candidates, n_features)`,
+                  the indexing refers to the samples in `candidates`.
         """
         # Validate input parameters.
         X, y, candidates, batch_size, return_utilities = self._validate_data(
@@ -148,11 +196,8 @@ def query(
         )
 
         X_cand, mapping = self._transform_candidates(candidates, X, y)
-
-        # Validate classifier type.
         check_type(fit_ensemble, "fit_ensemble", bool)
-
-        ensemble, est_arr, classes = _check_ensemble(
+        ensemble, est_arr, classes, sample_func, sample_dict = _check_ensemble(
             ensemble=ensemble,
             X=X,
             y=y,
@@ -160,34 +205,45 @@ def query(
             fit_ensemble=fit_ensemble,
             missing_label=self.missing_label_,
             estimator_types=[SkactivemlClassifier, SkactivemlRegressor],
+            sample_predictions_method_name=self.sample_predictions_method_name,
+            sample_predictions_dict=self.sample_predictions_dict,
         )
-
-        # Validate 'method'
         check_type(
             self.method,
             "method",
-            target_vals=["KL_divergence", "vote_entropy"],
+            target_vals=["KL_divergence", "vote_entropy", "variation_ratios"],
         )
 
-        # classes is None if the ensemble is a regressor
+        # `classes` is `None`, if `ensemble` is a regressor.
         if classes is not None:
             # Compute utilities.
             if self.method == "KL_divergence":
-                # probas = np.array(
-                #     [est.predict_proba(X_cand) for est in est_arr]
-                # )
-                probas = self._aggregate_predict_probas(
-                    X_cand, ensemble, est_arr
-                )
+                if sample_func is None:
+                    probas = self._aggregate_predict_probas(
+                        X_cand, ensemble, est_arr
+                    )
+                else:
+                    probas = sample_func(X_cand, **sample_dict)
                 utilities_cand = average_kl_divergence(probas, self.eps)
-            else:  # self.method == "vote_entropy":
-                votes = np.array([est.predict(X_cand) for est in est_arr]).T
-                utilities_cand = vote_entropy(votes, classes)
-
+            else:
+                if sample_func is None:
+                    votes = np.array(
+                        [est.predict(X_cand) for est in est_arr]
+                    ).T
+                else:
+                    probas = sample_func(X_cand, **sample_dict)
+                    votes = probas.argmax(axis=-1).T
+                if self.method == "vote_entropy":
+                    utilities_cand = vote_entropy(votes, classes)
+                else:
+                    utilities_cand = variation_ratios(votes)
         else:
-            results = np.array(
-                [learner.predict(X_cand) for learner in est_arr]
-            )
+            if sample_func is None:
+                results = np.array(
+                    [learner.predict(X_cand) for learner in est_arr]
+                )
+            else:
+                results = sample_func(X_cand, **sample_dict).T
             utilities_cand = np.std(results, axis=0)
 
         if mapping is None:
@@ -211,19 +267,19 @@ def _aggregate_predict_probas(self, X_cand, ensemble, est_arr):
         ----------
         X_cand : array-like of shape (n_samples, n_features)
             Samples whose probabilities are to be predicted.
-        ensemble : SkactivemlClassifier or list or tuple of
-        SkactivemlClassifier
-            If `ensemble` is a `SkactivemlClassifier`, it must have
-            `n_estimators` and `estimators_` after fitting as attribute. Then,
-            its estimators will be used as committee. If `ensemble` is
-            array-like, each element of this list must be
-            `SkactivemlClassifier` and will be used as committee member.
+        ensemble : SkactivemlClassifier or list or tuple of \
+                SkactivemlClassifier
+            - If `ensemble` is a `SkactivemlClassifier`, it must have
+              `n_estimators` and `estimators_` after fitting as attribute.
+              Then, its estimators will be used as committee.
+            - If `ensemble` is array-like, each element of this list must be
+              `SkactivemlClassifier` and will be used as committee member.
         est_arr : list or tuple of SkactivemlClassifier
             List of ensemble members contained in `ensemble`.
 
         Returns
         -------
-        probas: np.ndarray, shape (n_samples, n_classes)
+        probas : np.ndarray of shape (n_samples, n_classes)
             The mapped predicted probabilities.
         """
         if hasattr(ensemble, "classes_"):
@@ -256,21 +312,21 @@ def average_kl_divergence(probas, eps=1e-7):
 
     Parameters
     ----------
-    probas : array-like, shape (n_estimators, n_samples, n_classes)
+    probas : array-like of shape (n_estimators, n_samples, n_classes)
         The probability estimates of all estimators, samples, and classes.
     eps : float  > 0, optional (default=1e-7)
         Minimum probability threshold to compute log-probabilities.
 
     Returns
     -------
-    scores: np.ndarray, shape (n_samples)
+    scores : np.ndarray, shape (n_samples,)
         The Kullback-Leibler (KL) divergences.
 
     References
     ----------
-    [1] A. McCallum and K. Nigam. Employing EM in pool-based active learning
-        for text classification. In Proceedings of the International Conference
-        on Machine Learning (ICML), pages 359-367. Morgan Kaufmann, 1998.
+    .. [1] A. McCallum and K. Nigam. Employing EM in pool-based active learning
+       for text classification. In International Conference on Machine
+       Learning, pages 359-367, 1998.
     """
     # Check parameters.
     check_scalar(
@@ -315,14 +371,14 @@ def vote_entropy(votes, classes):
 
     Returns
     -------
-    vote_entropy : np.ndarray, shape (n_samples)
+    vote_entropy : np.ndarray of shape (n_samples,)
         The vote entropy of each row in `votes`.
 
     References
     ----------
-    [1] Engelson, Sean P., and Ido Dagan.
-        Minimizing manual annotation cost in supervised training from corpora.
-        arXiv preprint cmp-lg/9606030 (1996).
+    .. [1] Engelson, Sean P., and Ido Dagan. "Minimizing Manual Annotation Cost
+       in Supervised Training from Corpora." In Annual Meeting of the
+       Association for Computational Linguistics, pages 319-326, 1996.
     """
     # Check `votes` array.
     votes = check_array(votes)
@@ -341,6 +397,38 @@ def vote_entropy(votes, classes):
     return scores
 
 
+def variation_ratios(votes):
+    """Calculates the variation ratios for measuring the level of disagreement
+    in `QueryByCommittee`.
+
+    Parameters
+    ----------
+    votes : array-like of shape (n_samples, n_estimators)
+        The class predicted by the estimators for each sample.
+
+    Returns
+    -------
+    scores : np.ndarray of shape (n_samples,)
+        The variation ratios of each row in `votes`.
+
+    References
+    ----------
+    .. [1] Beluch, W. H., Genewein, T., Nürnberger, A., and Köhler, J. M.
+       The Power of Ensembles for Active Learning in Image Classification. In
+       Conference on Computer Vision and Pattern Recognition, pages 9368-9377,
+       2018.
+    """
+    # Check `votes` array.
+    votes = check_array(votes)
+    n_estimators = votes.shape[1]
+
+    # Count the votes.
+    vote_count = compute_vote_vectors(y=votes, missing_label=None)
+    scores = 1 - (vote_count.max(axis=-1) / n_estimators)
+
+    return scores
+
+
 def _check_ensemble(
     ensemble,
     estimator_types,
@@ -349,13 +437,19 @@ def _check_ensemble(
     sample_weight,
     fit_ensemble=True,
     missing_label=MISSING_LABEL,
+    sample_predictions_method_name=None,
+    sample_predictions_dict=None,
 ):
+    error_msg = (
+        f"`ensemble` must either be a `{estimator_types} "
+        f"with the attribute `n_ensembles` or `estimators_` or an array-like "
+        f"of {estimator_types} objects or implement a method with the name "
+        f"`{sample_predictions_method_name}`."
+    )
+
     # Check if the parameter `ensemble` is valid.
     for estimator_type in estimator_types:
-        if isinstance(ensemble, estimator_type) and (
-            hasattr(ensemble, "n_estimators")
-            or hasattr(ensemble, "estimators")
-        ):
+        if isinstance(ensemble, estimator_type):
             check_equal_missing_label(ensemble.missing_label, missing_label)
             # Fit the ensemble.
             if fit_ensemble:
@@ -366,23 +460,59 @@ def _check_ensemble(
             else:
                 check_is_fitted(ensemble)
 
-            if hasattr(ensemble, "estimators_"):
-                est_arr = ensemble.estimators_
+            if sample_predictions_method_name is not None:
+                check_type(
+                    sample_predictions_method_name,
+                    "sample_predictions_method_name",
+                    str,
+                )
+                if not hasattr(ensemble, sample_predictions_method_name):
+                    raise ValueError(
+                        "If `sample_predictions_method_name` is not `None`, "
+                        "`ensemble` must implement a method with this name."
+                    )
+                sample_func = getattr(ensemble, sample_predictions_method_name)
+                if sample_predictions_dict is None:
+                    sample_predictions_dict = {}
+                if not isinstance(sample_predictions_dict, dict):
+                    raise ValueError(
+                        "`sample_predictions_dict` must be a `dict`, if "
+                        "`sample_predictions_method_name` is not `None`."
+                    )
             else:
-                if hasattr(ensemble, "estimators"):
-                    n_estimators = len(ensemble.estimators)
-                else:
-                    n_estimators = ensemble.n_estimators
-                est_arr = [ensemble] * n_estimators
+                sample_func = None
+                if sample_predictions_dict is not None:
+                    raise ValueError(
+                        "`sample_predictions_dict` must be `None`, if "
+                        "`sample_predictions_method_name` is `None`."
+                    )
 
-            if estimator_type == SkactivemlClassifier:
-                return ensemble, est_arr, ensemble.classes_
+            if sample_func is not None:
+                est_arr = None
+            elif hasattr(ensemble, "estimators_"):
+                est_arr = ensemble.estimators_
+            elif hasattr(ensemble, "estimators"):
+                est_arr = [ensemble] * len(ensemble.estimators)
+            elif hasattr(ensemble, "n_estimators"):
+                est_arr = [ensemble] * ensemble.n_estimators
             else:
-                return ensemble, est_arr, None
+                raise TypeError(error_msg)
+
+            cls = getattr(ensemble, "classes_", None)
+            return ensemble, est_arr, cls, sample_func, sample_predictions_dict
 
         elif isinstance(ensemble, (list, tuple)) and isinstance(
             ensemble[0], estimator_type
         ):
+            if (
+                sample_predictions_dict is not None
+                or sample_predictions_method_name is not None
+            ):
+                raise ValueError(
+                    "`sample_predictions_method_name` and "
+                    "`sample_predictions_dict` must be `None`, if `ensemble` "
+                    "is array-like."
+                )
             est_arr = copy.deepcopy(ensemble)
             for i in range(len(est_arr)):
                 check_type(
@@ -409,13 +539,7 @@ def _check_ensemble(
                         f"parameter of each ensemble member to avoid "
                         f"this error.",
                     )
-            if estimator_type == SkactivemlClassifier:
-                return ensemble, est_arr, est_arr[0].classes_
-            else:
-                return ensemble, est_arr, None
+            cls = getattr(est_arr[0], "classes_", None)
+            return ensemble, est_arr, cls, None, None
 
-    raise TypeError(
-        f"`ensemble` must either be a `{estimator_types} "
-        f"with the attribute `n_ensembles` and `estimators_` after "
-        f"fitting or a list of {estimator_types} objects."
-    )
+    raise TypeError(error_msg)
diff --git a/skactiveml/utils/_multi_annot.py b/skactiveml/utils/_multi_annot.py
index a9ce4da6..e3086bcd 100644
--- a/skactiveml/utils/_multi_annot.py
+++ b/skactiveml/utils/_multi_annot.py
@@ -15,7 +15,7 @@ def ext_confusion_matrix(
 ):
     """Compute confusion matrix to evaluate the accuracy of a classification.
 
-    This is an extension of the 'sklearn.metric.confusion_matrix function' by
+    This is an extension of the `sklearn.metric.confusion_matrix` function by
     allowing missing labels and labels predicted by multiple annotators.
 
     By definition a confusion matrix :math:`C` is such that :math:`C_{i, j}`
@@ -34,8 +34,8 @@ def ext_confusion_matrix(
             Estimated targets as returned by multiple annotators.
     classes : array-like of shape (n_classes), default=None
         List of class labels to index the matrix. This may be used to reorder
-        or select a subset of labels. If ``None`` is given, those that appear
-        at least once in ``y_true`` or ``y_pred`` are used in sorted order.
+        or select a subset of labels. If `None` is given, those that appear
+        at least once in `y_true` or `y_pred` are used in sorted order.
     missing_label : {scalar, string, np.nan, None}, default=np.nan
         Value to represent a missing label.
     normalize : {'true', 'pred', 'all'}, default=None
@@ -47,7 +47,7 @@ def ext_confusion_matrix(
     -------
     conf_matrices : numpy.ndarray, shape (n_annotators, n_classes, n_classes)
         Confusion matrix whose i-th row and j-th column entry indicates the
-        number of samples with true label being i-th class and prediced label
+        number of samples with true label being i-th class and predicted label
         being j-th class.
 
     References
diff --git a/skactiveml/utils/_validation.py b/skactiveml/utils/_validation.py
index 2917663c..4a97ed32 100644
--- a/skactiveml/utils/_validation.py
+++ b/skactiveml/utils/_validation.py
@@ -334,7 +334,7 @@ def check_X_y(
           be infinite.
 
         .. versionadded:: 0.20
-           ``force_all_finite`` accepts the string ``'allow-nan'``.
+           `force_all_finite` accepts the string `'allow-nan'`.
 
         .. versionchanged:: 0.23
            Accepts `pd.NA` and converts it into `np.nan`
@@ -361,7 +361,7 @@ def check_X_y(
         Make sure that the 2D array has some minimum number of features
         (columns). The default value of 1 rejects empty datasets.
         This check is only enforced when X has effectively 2 dimensions or
-        is originally 1D and ``ensure_2d`` is True. Setting to 0 disables
+        is originally 1D and `ensure_2d` is True. Setting to 0 disables
         this check.
 
     y_numeric : boolean (default=False)


Test patch:
diff --git a/skactiveml/pool/tests/test_bald.py b/skactiveml/pool/tests/test_bald.py
index 08086385..e0dab043 100644
--- a/skactiveml/pool/tests/test_bald.py
+++ b/skactiveml/pool/tests/test_bald.py
@@ -87,6 +87,77 @@ def test_init_param_eps(self):
             test_cases,
         )
 
+    def test_init_param_sample_predictions_method_name(self):
+        # Fails as the default ensemble from `setup` does not support sampling.
+        test_cases = [
+            (0, TypeError),
+            (0.1, TypeError),
+            ("Test", ValueError),
+        ]
+        self._test_param("init", "sample_predictions_method_name", test_cases)
+        test_cases = [
+            ("predict_proba", ValueError),
+        ]
+        self._test_param(
+            "init",
+            "sample_predictions_method_name",
+            test_cases,
+            replace_query_params={
+                "ensemble": [
+                    ParzenWindowClassifier(),
+                    ParzenWindowClassifier(),
+                ]
+            },
+        )
+        test_cases = [
+            ("sample_proba", None),
+        ]
+        self._test_param(
+            "init",
+            "sample_predictions_method_name",
+            test_cases,
+            replace_query_params={
+                "ensemble": ParzenWindowClassifier(),
+                "fit_ensemble": True,
+            },
+        )
+
+    def test_init_param_sample_predictions_dict(self):
+        test_cases = [
+            (None, None),
+            ({}, None),
+            ({"n_samples": 1000}, None),
+            ("Test", ValueError),
+            ({"Test": 2}, TypeError),
+        ]
+        self._test_param(
+            "init",
+            "sample_predictions_dict",
+            test_cases,
+            replace_init_params={
+                "sample_predictions_method_name": "sample_proba",
+            },
+            replace_query_params={
+                "ensemble": ParzenWindowClassifier(),
+                "fit_ensemble": True,
+            },
+        )
+        test_cases = [
+            (None, None),
+            ({}, ValueError),
+            ({"n_samples": 1000}, ValueError),
+            ("Test", ValueError),
+            ({"Test": 2}, ValueError),
+        ]
+        self._test_param(
+            "init",
+            "sample_predictions_dict",
+            test_cases,
+            replace_init_params={
+                "sample_predictions_method_name": None,
+            },
+        )
+
     def test_query_param_ensemble(self, test_cases=None):
         test_cases = [] if test_cases is None else test_cases
         test_cases += [
diff --git a/skactiveml/pool/tests/test_query_by_committee.py b/skactiveml/pool/tests/test_query_by_committee.py
index a077af84..d88a3af6 100644
--- a/skactiveml/pool/tests/test_query_by_committee.py
+++ b/skactiveml/pool/tests/test_query_by_committee.py
@@ -19,6 +19,7 @@
     QueryByCommittee,
     average_kl_divergence,
     vote_entropy,
+    variation_ratios,
 )
 from skactiveml.regressor import NICKernelRegressor, SklearnRegressor
 from skactiveml.tests.template_query_strategy import (
@@ -75,7 +76,102 @@ def test_init_param_eps(self):
         ]
         self._test_param("init", "eps", test_cases, exclude_reg=True)
 
-    def test_query_param_ensemble(self, test_cases=None):
+    def test_init_param_sample_predictions_method_name(self):
+        # Fails as the default ensemble from `setup` does not support sampling.
+        test_cases = [
+            (0, TypeError),
+            (0.1, TypeError),
+            ("Test", ValueError),
+        ]
+        self._test_param("init", "sample_predictions_method_name", test_cases)
+        test_cases = [
+            ("predict_proba", ValueError),
+        ]
+        self._test_param(
+            "init",
+            "sample_predictions_method_name",
+            test_cases,
+            replace_query_params={
+                "ensemble": [
+                    ParzenWindowClassifier(),
+                    ParzenWindowClassifier(),
+                ]
+            },
+        )
+        test_cases = [
+            ("sample_proba", None),
+        ]
+        self._test_param(
+            "init",
+            "sample_predictions_method_name",
+            test_cases,
+            replace_query_params={
+                "ensemble": ParzenWindowClassifier(),
+                "fit_ensemble": True,
+            },
+        )
+        test_cases = [
+            ("sample_y", None),
+        ]
+        self._test_param(
+            "init",
+            "sample_predictions_method_name",
+            test_cases,
+            replace_query_params={
+                "ensemble": SklearnRegressor(GaussianProcessRegressor()),
+                "fit_ensemble": True,
+            },
+        )
+
+    def test_init_param_sample_predictions_dict(self):
+        test_cases = [
+            (None, None),
+            ({}, None),
+            ({"n_samples": 1000}, None),
+            ("Test", ValueError),
+            ({"Test": 2}, TypeError),
+        ]
+        self._test_param(
+            "init",
+            "sample_predictions_dict",
+            test_cases,
+            replace_init_params={
+                "sample_predictions_method_name": "sample_proba",
+            },
+            replace_query_params={
+                "ensemble": ParzenWindowClassifier(),
+                "fit_ensemble": True,
+            },
+        )
+        self._test_param(
+            "init",
+            "sample_predictions_dict",
+            test_cases,
+            replace_init_params={
+                "sample_predictions_method_name": "sample_y",
+            },
+            replace_query_params={
+                "ensemble": SklearnRegressor(GaussianProcessRegressor()),
+                "fit_ensemble": True,
+            },
+        )
+        test_cases = [
+            (None, None),
+            ({}, ValueError),
+            ({"n_samples": 1000}, ValueError),
+            ("Test", ValueError),
+            ({"Test": 2}, ValueError),
+        ]
+        self._test_param(
+            "init",
+            "sample_predictions_dict",
+            test_cases,
+            replace_init_params={
+                "sample_predictions_method_name": None,
+            },
+        )
+
+    def test_query_param_ensemble(self):
         estimators = [
             ("pwc1", ParzenWindowClassifier()),
             ("pwc2", ParzenWindowClassifier()),
@@ -99,7 +195,7 @@ def test_query_param_ensemble(self, test_cases=None):
             (None, TypeError),
             ("test", TypeError),
             (1, TypeError),
-            (ParzenWindowClassifier(classes=self.classes), TypeError),
+            (ParzenWindowClassifier(), TypeError),
             (GaussianProcessRegressor(), TypeError),
             (RandomForestRegressor(), TypeError),
             (RandomForestClassifier(), TypeError),
@@ -244,8 +340,8 @@ def test_query(self):
             query_params = deepcopy(self.query_default_params_clf)
             query_params["ensemble"] = ensemble
             query_params["return_utilities"] = True
-            for method in ["KL_divergence", "vote_entropy"]:
-                qs = QueryByCommittee(method=method)
+            for m in ["KL_divergence", "vote_entropy", "variation_ratios"]:
+                qs = QueryByCommittee(method=m)
                 idx, u = qs.query(**query_params)
                 self.assertEqual(len(idx), 1)
                 self.assertEqual(len(u), 1)
@@ -271,7 +367,7 @@ def test_average_kl_divergence(self):
         average_kl_divergence(np.full((10, 10, 10), 0.5))
         average_kl_divergence(np.zeros((10, 10, 10)))
         scores = average_kl_divergence(self.probas)
-        np.testing.assert_allclose(scores, self.scores)
+        np.testing.assert_almost_equal(scores, self.scores)
 
 
 class TestVoteEntropy(unittest.TestCase):
@@ -328,4 +424,32 @@ def test_param_classes(self):
 
     def test_vote_entropy(self):
         scores = vote_entropy(votes=self.votes, classes=self.classes)
-        np.testing.assert_array_equal(scores.round(10), self.scores.round(10))
+        np.testing.assert_almost_equal(scores, self.scores)
+
+
+class TestVariationRatios(unittest.TestCase):
+    def setUp(self):
+        self.votes = np.array(
+            [
+                [0, 0, 2],
+                [1, 0, 2],
+                [2, 1, 2],
+                [0, 0, 0],
+            ],
+        )
+        self.scores = np.array([1.0 / 3.0, 2.0 / 3.0, 1.0 / 3.0, 0.0])
+
+    def test_param_votes(self):
+        self.assertRaises(ValueError, variation_ratios, votes="string")
+        self.assertRaises(ValueError, variation_ratios, votes=1)
+        self.assertRaises(ValueError, variation_ratios, votes=[1])
+        self.assertRaises(ValueError, variation_ratios, votes=[[[1]]])
+        self.assertRaises(
+            ValueError,
+            variation_ratios,
+            votes=np.full((9, 9), np.nan),
+        )
+
+    def test_variation_ratios(self):
+        scores = variation_ratios(votes=self.votes)
+        np.testing.assert_almost_equal(scores, self.scores)
diff --git a/skactiveml/tests/template_estimator.py b/skactiveml/tests/template_estimator.py
index f4cd9ebb..ca43f141 100644
--- a/skactiveml/tests/template_estimator.py
+++ b/skactiveml/tests/template_estimator.py
@@ -6,6 +6,7 @@
     check_test_param_test_availability,
 )
 import numpy as np
+from sklearn.datasets import make_blobs
 from skactiveml.utils import (
     MISSING_LABEL,
 )
@@ -617,6 +618,43 @@ def test_predict_freq_param_X(self, test_cases=None):
             replace_fit_params=replace_fit_params,
         )
 
+    def test_sample_proba(self):
+        # Setup test cases.
+        X, y_full = make_blobs(n_samples=200, centers=4, random_state=0)
+        classes = np.unique(y_full)
+        pwc = self.estimator_class(
+            classes=classes, class_prior=1, missing_label=-1
+        )
+        y_missing = np.full_like(y_full, fill_value=-1)
+        y_partial_missing = y_full.copy()
+        y_partial_missing[30:50] = -1
+        y_class_0_missing = y_full.copy()
+        y_class_0_missing[y_full == 0] = -1
+
+        for y in [y_missing, y_partial_missing, y_class_0_missing, y_full]:
+            pwc.fit(X, y)
+
+            for n_samples in [1, 10]:
+                # Check shape of probabilities.
+                P_sampled = pwc.sample_proba(X, n_samples=n_samples)
+                shape_Expected = [n_samples, len(X), len(classes)]
+                np.testing.assert_array_equal(P_sampled.shape, shape_Expected)
+
+                # Check normalization of probabilities.
+                P_sums = P_sampled.sum(axis=-1)
+                P_sums_expected = np.ones_like(P_sums)
+                np.testing.assert_allclose(P_sums, P_sums_expected)
+
+        # Check value error if `alphas` as input to dirichlet are zero.
+        pwc = self.estimator_class(
+            classes=np.unique(y_full), class_prior=0, missing_label=-1
+        )
+        pwc.fit(X, y_missing)
+        self.assertRaises(ValueError, pwc.sample_proba, X=X, n_samples=10)
+
+        pwc.fit(X, y_class_0_missing)
+        self.assertRaises(ValueError, pwc.sample_proba, X=X, n_samples=10)
+
 
 class TemplateSkactivemlRegressor(TemplateEstimator):
     def setUp(
diff --git a/skactiveml/tests/test_base.py b/skactiveml/tests/test_base.py
index d4fa6574..357805e7 100644
--- a/skactiveml/tests/test_base.py
+++ b/skactiveml/tests/test_base.py
@@ -368,7 +368,7 @@ def test_update(self):
         )
 
 
-class ScaktivemlRegressorTest(unittest.TestCase):
+class SkactivemlRegressorTest(unittest.TestCase):
     @patch.multiple(SkactivemlRegressor, __abstractmethods__=set())
     def setUp(self):
         self.reg = SkactivemlRegressor(missing_label=-1)