Repository: catalyst-team/catalyst (language: python)

Issue:
albumentations augmentor



Discussion / hints:

Fix patch:
diff --git a/catalyst/callbacks/metrics/classification.py b/catalyst/callbacks/metrics/classification.py
index 4795cdf33f..38452bc4c7 100644
--- a/catalyst/callbacks/metrics/classification.py
+++ b/catalyst/callbacks/metrics/classification.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
 from catalyst.callbacks.metric import BatchMetricCallback
 from catalyst.metrics._classification import (
     MulticlassPrecisionRecallF1SupportMetric,
@@ -82,7 +84,7 @@ def __init__(
         self,
         input_key: str,
         target_key: str,
-        num_classes: int,
+        num_classes: Optional[int] = None,
         zero_division: int = 0,
         log_on_batch: bool = True,
         compute_per_class_metrics: bool = SETTINGS.compute_per_class_metrics,
@@ -92,11 +94,11 @@ def __init__(
         """Init."""
         super().__init__(
             metric=MulticlassPrecisionRecallF1SupportMetric(
-                num_classes=num_classes,
                 zero_division=zero_division,
                 prefix=prefix,
                 suffix=suffix,
                 compute_per_class_metrics=compute_per_class_metrics,
+                num_classes=num_classes,
             ),
             input_key=input_key,
             target_key=target_key,
@@ -186,7 +188,7 @@ def __init__(
         self,
         input_key: str,
         target_key: str,
-        num_classes: int,
+        num_classes: Optional[int] = None,
         zero_division: int = 0,
         log_on_batch: bool = True,
         compute_per_class_metrics: bool = SETTINGS.compute_per_class_metrics,
@@ -196,11 +198,11 @@ def __init__(
         """Init."""
         super().__init__(
             metric=MultilabelPrecisionRecallF1SupportMetric(
-                num_classes=num_classes,
                 zero_division=zero_division,
                 prefix=prefix,
                 suffix=suffix,
                 compute_per_class_metrics=compute_per_class_metrics,
+                num_classes=num_classes,
             ),
             input_key=input_key,
             target_key=target_key,
diff --git a/catalyst/metrics/_auc.py b/catalyst/metrics/_auc.py
index acfcf46ee5..c0e31ed081 100644
--- a/catalyst/metrics/_auc.py
+++ b/catalyst/metrics/_auc.py
@@ -178,7 +178,7 @@ def compute(self) -> Tuple[torch.Tensor, float, float, float]:
             scores = torch.cat(all_gather(scores))
             targets = torch.cat(all_gather(targets))
 
-        scores, targets, _ = process_multilabel_components(outputs=scores, targets=targets)
+        scores, targets, _, _ = process_multilabel_components(outputs=scores, targets=targets)
         per_class = auc(scores=scores, targets=targets)
         micro = binary_auc(scores=scores.view(-1), targets=targets.view(-1))[0]
         macro = per_class.mean().item()
diff --git a/catalyst/metrics/_classification.py b/catalyst/metrics/_classification.py
index 4a53a12653..41047215ba 100644
--- a/catalyst/metrics/_classification.py
+++ b/catalyst/metrics/_classification.py
@@ -1,6 +1,5 @@
 from typing import Any, Dict, List, Optional, Tuple, Union
 from collections import defaultdict
-from functools import partial
 
 import numpy as np
 
@@ -21,16 +20,12 @@
     import torch_xla.core.xla_model as xm
 
 
-class StatisticsMetric(ICallbackBatchMetric):
+class BinaryStatisticsMetric(ICallbackBatchMetric):
     """
     This metric accumulates true positive, false positive, true negative,
-    false negative, support statistics from data.
-
-    It can work in binary, multiclass and multilabel classification tasks.
+    false negative, support statistics from binary data.
 
     Args:
-        mode: one of "binary", "multiclass" and "multilabel"
-        num_classes: number of classes
         compute_on_call: if True, computes and returns metric value during metric call
         prefix: metric prefix
         suffix: metric suffix
@@ -39,29 +34,23 @@ class StatisticsMetric(ICallbackBatchMetric):
         ValueError: if mode is incorrect
 
     Examples:
-
     .. code-block:: python
-
         import torch
         from torch.utils.data import DataLoader, TensorDataset
         from catalyst import dl
-
         # sample data
         num_samples, num_features, num_classes = int(1e4), int(1e1), 4
         X = torch.rand(num_samples, num_features)
         y = (torch.rand(num_samples,) * num_classes).to(torch.int64)
-
         # pytorch loaders
         dataset = TensorDataset(X, y)
         loader = DataLoader(dataset, batch_size=32, num_workers=1)
         loaders = {"train": loader, "valid": loader}
-
         # model, criterion, optimizer, scheduler
         model = torch.nn.Linear(num_features, num_classes)
         criterion = torch.nn.CrossEntropyLoss()
         optimizer = torch.optim.Adam(model.parameters())
         scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [2])
-
         # model training
         runner = dl.SupervisedRunner(
             input_key="features", output_key="logits", target_key="targets", loss_key="loss"
@@ -88,34 +77,22 @@ class StatisticsMetric(ICallbackBatchMetric):
                 dl.AUCCallback(input_key="logits", target_key="targets"),
             ],
         )
-
     .. note::
         Please follow the `minimal examples`_ sections for more use cases.
-
         .. _`minimal examples`: https://github.com/catalyst-team/catalyst#minimal-examples
+
     """
 
     def __init__(
         self,
-        mode: str,
-        num_classes: int = None,
         compute_on_call: bool = True,
         prefix: Optional[str] = None,
         suffix: Optional[str] = None,
     ):
         """Init params"""
         super().__init__(compute_on_call=compute_on_call, prefix=prefix, suffix=suffix)
-        if mode == "binary":
-            self.statistics_fn = get_binary_statistics
-        elif mode == "multiclass":
-            self.statistics_fn = partial(get_multiclass_statistics, num_classes=num_classes)
-        elif mode == "multilabel":
-            self.statistics_fn = get_multilabel_statistics
-        else:
-            raise ValueError("Mode should be one of 'binary', 'multiclass', 'multilabel'")
-
-        self.num_classes = num_classes
         self.statistics = None
+        self.num_classes = 2
         self._ddp_backend = None
         self.reset()
 
@@ -141,8 +118,9 @@ def update(
         Returns:
             Tuple of int or array: true negative, false positive, false
                 negative, true positive and support statistics
+
         """
-        tn, fp, fn, tp, support = self.statistics_fn(
+        tn, fp, fn, tp, support = get_binary_statistics(
             outputs=outputs.cpu().detach(), targets=targets.cpu().detach()
         )
 
@@ -170,6 +148,7 @@ def update_key_value(self, outputs: torch.Tensor, targets: torch.Tensor) -> Dict
 
         Returns:
             dict of statistics for current input
+
         """
         tn, fp, fn, tp, support = self.update(outputs=outputs, targets=targets)
         return {"fn": fn, "fp": fp, "support": support, "tn": tn, "tp": tp}
@@ -180,6 +159,7 @@ def compute(self) -> Dict[str, Union[int, np.array]]:
 
         Returns:
             dict of statistics
+
         """
         return self.statistics
 
@@ -191,203 +171,366 @@ def compute_key_value(self) -> Dict[str, float]:
             dict of statistics
 
         Examples:
-            >>> For binary mode: {"tp": 3, "fp": 4, "tn": 5, "fn": 1, "support": 13}
-            >>> For other modes: {"tp": np.array([1, 2, 1]), "fp": np.array([2, 1, 0]), ...}
+            >>> {"tp": 3, "fp": 4, "tn": 5, "fn": 1, "support": 13}
+
         """
         result = self.compute()
         return {k: result[k] for k in sorted(result.keys())}
 
 
-class PrecisionRecallF1SupportMetric(StatisticsMetric):
+class MulticlassStatisticsMetric(ICallbackBatchMetric):
     """
-    Metric that can collect statistics and count precision, recall, f1_score and support with it.
+    This metric accumulates true positive, false positive, true negative,
+    false negative, support statistics from multiclass data.
 
     Args:
-        mode: one of "binary", "multiclass" and "multilabel"
-        num_classes: number of classes in loader's dataset
-        zero_division: value to set in case of zero division during metrics
-            (precision, recall) computation; should be one of 0 or 1
-        compute_on_call: if True, allows compute metric's value on call
-        compute_per_class_metrics: boolean flag to compute per-class metrics
-            (default: SETTINGS.compute_per_class_metrics or False).
-        prefix: metrics prefix
-        suffix: metrics suffix
+        compute_on_call: if True, computes and returns metric value during metric call
+        prefix: metric prefix
+        suffix: metric suffix
+        num_classes: number of classes
+
+    Raises:
+        ValueError: if mode is incorrect
+
+    Examples:
+    .. code-block:: python
+        import torch
+        from torch.utils.data import DataLoader, TensorDataset
+        from catalyst import dl
+        # sample data
+        num_samples, num_features, num_classes = int(1e4), int(1e1), 4
+        X = torch.rand(num_samples, num_features)
+        y = (torch.rand(num_samples,) * num_classes).to(torch.int64)
+        # pytorch loaders
+        dataset = TensorDataset(X, y)
+        loader = DataLoader(dataset, batch_size=32, num_workers=1)
+        loaders = {"train": loader, "valid": loader}
+        # model, criterion, optimizer, scheduler
+        model = torch.nn.Linear(num_features, num_classes)
+        criterion = torch.nn.CrossEntropyLoss()
+        optimizer = torch.optim.Adam(model.parameters())
+        scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [2])
+        # model training
+        runner = dl.SupervisedRunner(
+            input_key="features", output_key="logits", target_key="targets", loss_key="loss"
+        )
+        runner.train(
+            model=model,
+            criterion=criterion,
+            optimizer=optimizer,
+            scheduler=scheduler,
+            loaders=loaders,
+            logdir="./logdir",
+            num_epochs=3,
+            valid_loader="valid",
+            valid_metric="accuracy03",
+            minimize_valid_metric=False,
+            verbose=True,
+            callbacks=[
+                dl.AccuracyCallback(
+                    input_key="logits", target_key="targets", num_classes=num_classes
+                ),
+                dl.PrecisionRecallF1SupportCallback(
+                    input_key="logits", target_key="targets", num_classes=num_classes
+                ),
+                dl.AUCCallback(input_key="logits", target_key="targets"),
+            ],
+        )
+    .. note::
+        Please follow the `minimal examples`_ sections for more use cases.
+        .. _`minimal examples`: https://github.com/catalyst-team/catalyst#minimal-examples
+
     """
 
     def __init__(
         self,
-        mode: str,
-        num_classes: int = None,
-        zero_division: int = 0,
         compute_on_call: bool = True,
-        compute_per_class_metrics: bool = SETTINGS.compute_per_class_metrics,
-        prefix: str = None,
-        suffix: str = None,
-    ) -> None:
-        """Init PrecisionRecallF1SupportMetric instance"""
-        super().__init__(
-            compute_on_call=compute_on_call,
-            prefix=prefix,
-            suffix=suffix,
-            num_classes=num_classes,
-            mode=mode,
-        )
-        self.compute_per_class_metrics = compute_per_class_metrics
-        self.zero_division = zero_division
+        prefix: Optional[str] = None,
+        suffix: Optional[str] = None,
+        num_classes: Optional[int] = None,
+    ):
+        """Init params"""
+        super().__init__(compute_on_call=compute_on_call, prefix=prefix, suffix=suffix)
+        self.statistics = None
+        self.num_classes = num_classes
+        self._ddp_backend = None
         self.reset()
 
-    def _convert_metrics_to_kv(self, per_class, micro, macro, weighted) -> Dict[str, float]:
-        """
-        Convert metrics aggregation to key-value format
-
-        Args:
-            per_class: per-class metrics, array of shape (4, self.num_classes)
-                of precision, recall, f1 and support metrics
-            micro: micro averaged metrics, array of shape (self.num_classes)
-                of precision, recall, f1 and support metrics
-            macro: macro averaged metrics, array of shape (self.num_classes)
-                of precision, recall, f1 and support metrics
-            weighted: weighted averaged metrics, array of shape (self.num_classes)
-                of precision, recall, f1 and support metrics
-
-        Returns:
-            dict of key-value metrics
-        """
-        kv_metrics = {}
-        for aggregation_name, aggregated_metrics in zip(
-            ("_micro", "_macro", "_weighted"), (micro, macro, weighted)
-        ):
-            metrics = {
-                f"{metric_name}/{aggregation_name}": metric_value
-                for metric_name, metric_value in zip(
-                    ("precision", "recall", "f1"), aggregated_metrics[:-1]
-                )
-            }
-            kv_metrics.update(metrics)
+    # multiprocessing could not handle lamdas, so..
+    def _mp_hack(self):
+        return np.zeros(shape=(self.num_classes,))
 
-        # @TODO: rewrite this block - should be without `num_classes`
-        if self.compute_per_class_metrics:
-            per_class_metrics = {
-                f"{metric_name}/class_{i:02d}": metric_value[i]
-                for metric_name, metric_value in zip(
-                    ("precision", "recall", "f1", "support"), per_class
-                )
-                for i in range(self.num_classes)
-            }
-            kv_metrics.update(per_class_metrics)
-        return kv_metrics
+    def reset(self) -> None:
+        """Reset all the statistics."""
+        self.statistics = defaultdict(self._mp_hack)
+        self._ddp_backend = get_backend()
 
-    def update(self, outputs: torch.Tensor, targets: torch.Tensor) -> Tuple[Any, Any, Any, Any]:
+    def update(
+        self, outputs: torch.Tensor, targets: torch.Tensor
+    ) -> Union[Tuple[int, int, int, int, int, int], Tuple[Any, Any, Any, Any, Any, int]]:
         """
-        Update statistics and return intermediate metrics results
+        Compute statistics from outputs and targets, update accumulated statistics with new values.
 
         Args:
             outputs: prediction values
             targets: true answers
 
         Returns:
-            tuple of metrics intermediate results with per-class, micro, macro and
-                weighted averaging
+            Tuple of int or array: true negative, false positive, false
+                negative, true positive, support statistics and num_classes
+
         """
-        tn, fp, fn, tp, support = super().update(outputs=outputs, targets=targets)
-        per_class, micro, macro, weighted = get_aggregated_metrics(
-            tp=tp, fp=fp, fn=fn, support=support, zero_division=self.zero_division
+        tn, fp, fn, tp, support, num_classes = get_multiclass_statistics(
+            outputs=outputs.cpu().detach(),
+            targets=targets.cpu().detach(),
+            num_classes=self.num_classes,
         )
-        return per_class, micro, macro, weighted
+
+        tn = tn.numpy()
+        fp = fp.numpy()
+        fn = fn.numpy()
+        tp = tp.numpy()
+        support = support.numpy()
+
+        if self.num_classes is None:
+            self.num_classes = num_classes
+
+        self.statistics["tn"] += tn
+        self.statistics["fp"] += fp
+        self.statistics["fn"] += fn
+        self.statistics["tp"] += tp
+        self.statistics["support"] += support
+
+        return tn, fp, fn, tp, support, self.num_classes
 
     def update_key_value(self, outputs: torch.Tensor, targets: torch.Tensor) -> Dict[str, float]:
         """
-        Update statistics and return intermediate metrics results
+        Update statistics and return statistics intermediate result
 
         Args:
             outputs: prediction values
             targets: true answers
 
         Returns:
-            dict of metrics intermediate results
+            dict of statistics for current input
+
         """
-        per_class, micro, macro, weighted = self.update(outputs=outputs, targets=targets)
-        metrics = self._convert_metrics_to_kv(
-            per_class=per_class, micro=micro, macro=macro, weighted=weighted
-        )
-        return metrics
+        tn, fp, fn, tp, support, _ = self.update(outputs=outputs, targets=targets)
+        return {"fn": fn, "fp": fp, "support": support, "tn": tn, "tp": tp}
 
-    def compute(self) -> Any:
+    def compute(self) -> Dict[str, Union[int, np.array]]:
         """
-        Compute precision, recall, f1 score and support.
-        Compute micro, macro and weighted average for the metrics.
+        Return accumulated statistics
 
         Returns:
-            list of aggregated metrics: per-class, micro, macro and weighted averaging of
-                precision, recall, f1 score and support metrics
-        """
-        # ddp hotfix, could be done better
-        # but metric must handle DDP on it's own
-        if self._ddp_backend == "xla":
-            device = get_device()
-            for key in self.statistics:
-                key_statistics = torch.tensor([self.statistics[key]], device=device)
-                key_statistics = xm.all_gather(key_statistics).sum(dim=0).cpu().numpy()
-                self.statistics[key] = key_statistics
-        elif self._ddp_backend == "ddp":
-            for key in self.statistics:
-                value: List[np.ndarray] = all_gather(self.statistics[key])
-                value: np.ndarray = np.sum(np.vstack(value), axis=0)
-                self.statistics[key] = value
+            dict of statistics
 
-        per_class, micro, macro, weighted = get_aggregated_metrics(
-            tp=self.statistics["tp"],
-            fp=self.statistics["fp"],
-            fn=self.statistics["fn"],
-            support=self.statistics["support"],
-            zero_division=self.zero_division,
-        )
-        if self.compute_per_class_metrics:
-            return per_class, micro, macro, weighted
-        else:
-            return [], micro, macro, weighted
+        """
+        return self.statistics
 
     def compute_key_value(self) -> Dict[str, float]:
         """
-        Compute precision, recall, f1 score and support.
-        Compute micro, macro and weighted average for the metrics.
+        Return accumulated statistics
 
         Returns:
-            dict of metrics
+            dict of statistics
+
+        Examples:
+            >>> {"tp": np.array([1, 2, 1]), "fp": np.array([2, 1, 0]), ...}
+
         """
-        per_class, micro, macro, weighted = self.compute()
-        metrics = self._convert_metrics_to_kv(
-            per_class=per_class, micro=micro, macro=macro, weighted=weighted
-        )
-        return metrics
+        result = self.compute()
+        return {k: result[k] for k in sorted(result.keys())}
 
 
-class BinaryPrecisionRecallF1Metric(StatisticsMetric):
-    """Precision, recall, f1_score and support metrics for binary classification.
+class MultilabelStatisticsMetric(ICallbackBatchMetric):
+    """
+    This metric accumulates true positive, false positive, true negative,
+    false negative, support statistics from multilabel data.
 
     Args:
-        zero_division: value to set in case of zero division during metrics
-            (precision, recall) computation; should be one of 0 or 1
-        compute_on_call: if True, allows compute metric's value on call
+        compute_on_call: if True, computes and returns metric value during metric call
         prefix: metric prefix
         suffix: metric suffix
-    """
+        num_classes: number of classes
 
-    def __init__(
-        self,
-        zero_division: int = 0,
-        compute_on_call: bool = True,
-        prefix: Optional[str] = None,
-        suffix: Optional[str] = None,
-    ):
-        """Init BinaryPrecisionRecallF1SupportMetric instance"""
-        super().__init__(
-            num_classes=2,
-            mode="binary",
+    Raises:
+        ValueError: if mode is incorrect
+
+    Examples:
+    .. code-block:: python
+        import torch
+        from torch.utils.data import DataLoader, TensorDataset
+        from catalyst import dl
+        # sample data
+        num_samples, num_features, num_classes = int(1e4), int(1e1), 4
+        X = torch.rand(num_samples, num_features)
+        y = (torch.rand(num_samples,) * num_classes).to(torch.int64)
+        # pytorch loaders
+        dataset = TensorDataset(X, y)
+        loader = DataLoader(dataset, batch_size=32, num_workers=1)
+        loaders = {"train": loader, "valid": loader}
+        # model, criterion, optimizer, scheduler
+        model = torch.nn.Linear(num_features, num_classes)
+        criterion = torch.nn.CrossEntropyLoss()
+        optimizer = torch.optim.Adam(model.parameters())
+        scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [2])
+        # model training
+        runner = dl.SupervisedRunner(
+            input_key="features", output_key="logits", target_key="targets", loss_key="loss"
+        )
+        runner.train(
+            model=model,
+            criterion=criterion,
+            optimizer=optimizer,
+            scheduler=scheduler,
+            loaders=loaders,
+            logdir="./logdir",
+            num_epochs=3,
+            valid_loader="valid",
+            valid_metric="accuracy03",
+            minimize_valid_metric=False,
+            verbose=True,
+            callbacks=[
+                dl.AccuracyCallback(
+                    input_key="logits", target_key="targets", num_classes=num_classes
+                ),
+                dl.PrecisionRecallF1SupportCallback(
+                    input_key="logits", target_key="targets", num_classes=num_classes
+                ),
+                dl.AUCCallback(input_key="logits", target_key="targets"),
+            ],
+        )
+    .. note::
+        Please follow the `minimal examples`_ sections for more use cases.
+        .. _`minimal examples`: https://github.com/catalyst-team/catalyst#minimal-examples
+
+    """
+
+    def __init__(
+        self,
+        compute_on_call: bool = True,
+        prefix: Optional[str] = None,
+        suffix: Optional[str] = None,
+        num_classes: Optional[int] = None,
+    ):
+        """Init params"""
+        super().__init__(compute_on_call=compute_on_call, prefix=prefix, suffix=suffix)
+        self.statistics = None
+        self.num_classes = num_classes
+        self._ddp_backend = None
+        self.reset()
+
+    # multiprocessing could not handle lamdas, so..
+    def _mp_hack(self):
+        return np.zeros(shape=(self.num_classes,))
+
+    def reset(self) -> None:
+        """Reset all the statistics."""
+        self.statistics = defaultdict(self._mp_hack)
+        self._ddp_backend = get_backend()
+
+    def update(
+        self, outputs: torch.Tensor, targets: torch.Tensor
+    ) -> Union[Tuple[int, int, int, int, int, int], Tuple[Any, Any, Any, Any, Any, int]]:
+        """
+        Compute statistics from outputs and targets, update accumulated statistics with new values.
+
+        Args:
+            outputs: prediction values
+            targets: true answers
+
+        Returns:
+            Tuple of int or array: true negative, false positive, false
+                negative, true positive, support statistics and num_classes
+
+        """
+        tn, fp, fn, tp, support, num_classes = get_multilabel_statistics(
+            outputs=outputs.cpu().detach(), targets=targets.cpu().detach()
+        )
+
+        tn = tn.numpy()
+        fp = fp.numpy()
+        fn = fn.numpy()
+        tp = tp.numpy()
+        support = support.numpy()
+        if self.num_classes is None:
+            self.num_classes = num_classes
+
+        self.statistics["tn"] += tn
+        self.statistics["fp"] += fp
+        self.statistics["fn"] += fn
+        self.statistics["tp"] += tp
+        self.statistics["support"] += support
+
+        return tn, fp, fn, tp, support, self.num_classes
+
+    def update_key_value(self, outputs: torch.Tensor, targets: torch.Tensor) -> Dict[str, float]:
+        """
+        Update statistics and return statistics intermediate result
+
+        Args:
+            outputs: prediction values
+            targets: true answers
+
+        Returns:
+            dict of statistics for current input
+
+        """
+        tn, fp, fn, tp, support, _ = self.update(outputs=outputs, targets=targets)
+        return {"fn": fn, "fp": fp, "support": support, "tn": tn, "tp": tp}
+
+    def compute(self) -> Dict[str, Union[int, np.array]]:
+        """
+        Return accumulated statistics
+
+        Returns:
+            dict of statistics
+
+        """
+        return self.statistics
+
+    def compute_key_value(self) -> Dict[str, float]:
+        """
+        Return accumulated statistics
+
+        Returns:
+            dict of statistics
+
+        Examples:
+            >>> {"tp": np.array([1, 2, 1]), "fp": np.array([2, 1, 0]), ...}
+
+        """
+        result = self.compute()
+        return {k: result[k] for k in sorted(result.keys())}
+
+
+class BinaryPrecisionRecallF1Metric(BinaryStatisticsMetric):
+    """Precision, recall, f1_score and support metrics for binary classification.
+
+    Args:
+        zero_division: value to set in case of zero division during metrics
+            (precision, recall) computation; should be one of 0 or 1
+        compute_on_call: if True, allows compute metric's value on call
+        prefix: metric prefix
+        suffix: metric suffix
+
+    """
+
+    def __init__(
+        self,
+        zero_division: int = 0,
+        compute_on_call: bool = True,
+        prefix: Optional[str] = None,
+        suffix: Optional[str] = None,
+    ):
+        """Init BinaryPrecisionRecallF1SupportMetric instance"""
+        super().__init__(
             compute_on_call=compute_on_call,
             prefix=prefix,
             suffix=suffix,
         )
+        self.statistics = None
         self.zero_division = zero_division
         self.reset()
 
@@ -405,6 +548,7 @@ def _convert_metrics_to_kv(
 
         Returns:
             dict of metrics
+
         """
         kv_metrics = {
             "precision": precision_value,
@@ -427,6 +571,7 @@ def update(self, outputs: torch.Tensor, targets: torch.Tensor) -> Tuple[float, f
 
         Returns:
             tuple of intermediate metrics: precision, recall, f1 score
+
         """
         tn, fp, fn, tp, support = super().update(outputs=outputs, targets=targets)
         precision_value, recall_value, f1_value = get_binary_metrics(
@@ -444,6 +589,7 @@ def update_key_value(self, outputs: torch.Tensor, targets: torch.Tensor) -> Dict
 
         Returns:
             dict of intermediate metrics
+
         """
         precision_value, recall_value, f1_value = self.update(outputs=outputs, targets=targets)
         kv_metrics = self._convert_metrics_to_kv(
@@ -457,6 +603,7 @@ def compute(self) -> Tuple[float, float, float]:
 
         Returns:
             tuple of metrics: precision, recall, f1 score
+
         """
         # ddp hotfix, could be done better
         # but metric must handle DDP on it's own
@@ -482,6 +629,7 @@ def compute_key_value(self) -> Dict[str, float]:
 
         Returns:
             dict of metrics
+
         """
         precision_value, recall_value, f1_value = self.compute()
         kv_metrics = self._convert_metrics_to_kv(
@@ -490,369 +638,340 @@ def compute_key_value(self) -> Dict[str, float]:
         return kv_metrics
 
 
-class MulticlassPrecisionRecallF1SupportMetric(PrecisionRecallF1SupportMetric):
+class MulticlassPrecisionRecallF1SupportMetric(MulticlassStatisticsMetric):
     """
-    Precision, recall, f1_score and support metrics for multiclass classification.
-    Counts metrics with macro, micro and weighted average.
+    Metric that can collect statistics and count precision, recall, f1_score and support with it.
 
     Args:
-        num_classes: number of classes in loader's dataset
         zero_division: value to set in case of zero division during metrics
             (precision, recall) computation; should be one of 0 or 1
         compute_on_call: if True, allows compute metric's value on call
         compute_per_class_metrics: boolean flag to compute per-class metrics
             (default: SETTINGS.compute_per_class_metrics or False).
-        prefix: metric prefix
-        suffix: metric suffix
-
-    Examples:
-
-    .. code-block:: python
-
-        import torch
-        from catalyst import metrics
-
-        num_classes = 4
-        zero_division = 0
-        outputs_list = [torch.tensor([0, 1, 2]), torch.tensor([2, 3]), torch.tensor([0, 1, 3])]
-        targets_list = [torch.tensor([0, 1, 1]), torch.tensor([2, 3]), torch.tensor([0, 1, 2])]
-
-        metric = metrics.MulticlassPrecisionRecallF1SupportMetric(
-            num_classes=num_classes, zero_division=zero_division
-        )
-        metric.reset()
-
-        for outputs, targets in zip(outputs_list, targets_list):
-            metric.update(outputs=outputs, targets=targets)
-
-        metric.compute()
-        # (
-        #     # per class precision, recall, f1, support
-        #     (
-        #         array([1. , 1. , 0.5, 0.5]),
-        #         array([1.        , 0.66666667, 0.5       , 1.        ]),
-        #         array([0.999995  , 0.7999952 , 0.499995  , 0.66666222]),
-        #         array([2., 3., 2., 1.]),
-        #     ),
-        #     # micro precision, recall, f1, support
-        #     (0.75, 0.75, 0.7499950000333331, None),
-        #     # macro precision, recall, f1, support
-        #     (0.75, 0.7916666666666667, 0.7416618555889127, None),
-        #     # weighted precision, recall, f1, support
-        #     (0.8125, 0.75, 0.7583284778110313, None)
-        # )
-
-        metric.compute_key_value()
-        # {
-        #     'f1/_macro': 0.7416618555889127,
-        #     'f1/_micro': 0.7499950000333331,
-        #     'f1/_weighted': 0.7583284778110313,
-        #     'f1/class_00': 0.9999950000249999,
-        #     'f1/class_01': 0.7999952000287999,
-        #     'f1/class_02': 0.49999500004999947,
-        #     'f1/class_03': 0.6666622222518517,
-        #     'precision/_macro': 0.75,
-        #     'precision/_micro': 0.75,
-        #     'precision/_weighted': 0.8125,
-        #     'precision/class_00': 1.0,
-        #     'precision/class_01': 1.0,
-        #     'precision/class_02': 0.5,
-        #     'precision/class_03': 0.5,
-        #     'recall/_macro': 0.7916666666666667,
-        #     'recall/_micro': 0.75,
-        #     'recall/_weighted': 0.75,
-        #     'recall/class_00': 1.0,
-        #     'recall/class_01': 0.6666666666666667,
-        #     'recall/class_02': 0.5,
-        #     'recall/class_03': 1.0,
-        #     'support/class_00': 2.0,
-        #     'support/class_01': 3.0,
-        #     'support/class_02': 2.0,
-        #     'support/class_03': 1.0
-        # }
-
-        metric.reset()
-        metric(outputs_list[0], targets_list[0])
-        # (
-        #     # per class precision, recall, f1, support
-        #     (
-        #         array([1., 1., 0., 0.]),
-        #         array([1. , 0.5, 0. , 0. ]),
-        #         array([0.999995  , 0.66666222, 0.        , 0.        ]),
-        #         array([1., 2., 0., 0.]),
-        #     ),
-        #     # micro precision, recall, f1, support
-        #     (0.6666666666666667, 0.6666666666666667, 0.6666616667041664, None),
-        #     # macro precision, recall, f1, support
-        #     (0.5, 0.375, 0.41666430556921286, None),
-        #     # weighted precision, recall, f1, support
-        #     (1.0, 0.6666666666666666, 0.7777731481762343, None)
-        # )
-
-    .. code-block:: python
-
-        import torch
-        from torch.utils.data import DataLoader, TensorDataset
-        from catalyst import dl
-
-        # sample data
-        num_samples, num_features, num_classes = int(1e4), int(1e1), 4
-        X = torch.rand(num_samples, num_features)
-        y = (torch.rand(num_samples,) * num_classes).to(torch.int64)
-
-        # pytorch loaders
-        dataset = TensorDataset(X, y)
-        loader = DataLoader(dataset, batch_size=32, num_workers=1)
-        loaders = {"train": loader, "valid": loader}
-
-        # model, criterion, optimizer, scheduler
-        model = torch.nn.Linear(num_features, num_classes)
-        criterion = torch.nn.CrossEntropyLoss()
-        optimizer = torch.optim.Adam(model.parameters())
-        scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [2])
-
-        # model training
-        runner = dl.SupervisedRunner(
-            input_key="features", output_key="logits", target_key="targets", loss_key="loss"
-        )
-        runner.train(
-            model=model,
-            criterion=criterion,
-            optimizer=optimizer,
-            scheduler=scheduler,
-            loaders=loaders,
-            logdir="./logdir",
-            num_epochs=3,
-            valid_loader="valid",
-            valid_metric="accuracy03",
-            minimize_valid_metric=False,
-            verbose=True,
-            callbacks=[
-                dl.AccuracyCallback(
-                    input_key="logits", target_key="targets", num_classes=num_classes
-                ),
-                dl.PrecisionRecallF1SupportCallback(
-                    input_key="logits", target_key="targets", num_classes=num_classes
-                ),
-                dl.AUCCallback(input_key="logits", target_key="targets"),
-            ],
-        )
-
-    .. note::
-        Please follow the `minimal examples`_ sections for more use cases.
+        prefix: metrics prefix
+        suffix: metrics suffix
+        num_classes: number of classes
 
-        .. _`minimal examples`: https://github.com/catalyst-team/catalyst#minimal-examples
     """
 
     def __init__(
         self,
-        num_classes: int = None,
         zero_division: int = 0,
         compute_on_call: bool = True,
         compute_per_class_metrics: bool = SETTINGS.compute_per_class_metrics,
-        prefix: Optional[str] = None,
-        suffix: Optional[str] = None,
-    ):
-        """Init MultiClassPrecisionRecallF1SupportMetric instance"""
+        prefix: str = None,
+        suffix: str = None,
+        num_classes: Optional[int] = None,
+    ) -> None:
+        """Init PrecisionRecallF1SupportMetric instance"""
         super().__init__(
-            compute_on_call=compute_on_call,
-            compute_per_class_metrics=compute_per_class_metrics,
-            prefix=prefix,
-            suffix=suffix,
-            num_classes=num_classes,
-            zero_division=zero_division,
-            mode="multiclass",
+            compute_on_call=compute_on_call, prefix=prefix, suffix=suffix, num_classes=num_classes
         )
+        self.compute_per_class_metrics = compute_per_class_metrics
+        self.zero_division = zero_division
+        self.num_classes = num_classes
+        self.reset()
 
+    def _convert_metrics_to_kv(self, per_class, micro, macro, weighted) -> Dict[str, float]:
+        """
+        Convert metrics aggregation to key-value format
 
-class MultilabelPrecisionRecallF1SupportMetric(PrecisionRecallF1SupportMetric):
-    """
-    Precision, recall, f1_score and support metrics for multilabel classification.
-    Counts metrics with macro, micro and weighted average.
-
-    Args:
-        num_classes: number of classes in loader's dataset
-        zero_division: value to set in case of zero division during metrics
-            (precision, recall) computation; should be one of 0 or 1
-        compute_on_call: if True, allows compute metric's value on call
-        compute_per_class_metrics: boolean flag to compute per-class metrics
-            (default: SETTINGS.compute_per_class_metrics or False).
-        prefix: metric prefix
-        suffix: metric suffix
+        Args:
+            per_class: per-class metrics, array of shape (4, self.num_classes)
+                of precision, recall, f1 and support metrics
+            micro: micro averaged metrics, array of shape (self.num_classes)
+                of precision, recall, f1 and support metrics
+            macro: macro averaged metrics, array of shape (self.num_classes)
+                of precision, recall, f1 and support metrics
+            weighted: weighted averaged metrics, array of shape (self.num_classes)
+                of precision, recall, f1 and support metrics
 
-    Examples:
+        Returns:
+            dict of key-value metrics
 
-    .. code-block:: python
+        """
+        kv_metrics = {}
+        for aggregation_name, aggregated_metrics in zip(
+            ("_micro", "_macro", "_weighted"), (micro, macro, weighted)
+        ):
+            metrics = {
+                f"{metric_name}/{aggregation_name}": metric_value
+                for metric_name, metric_value in zip(
+                    ("precision", "recall", "f1"), aggregated_metrics[:-1]
+                )
+            }
+            kv_metrics.update(metrics)
 
-        import torch
-        from catalyst import metrics
-
-        num_classes = 4
-        zero_division = 0
-        outputs_list = [
-            torch.tensor([[0, 1, 0, 1], [0, 0, 0, 0], [0, 1, 1, 0]]),
-            torch.tensor([[0, 1, 1, 1], [0, 0, 0, 1], [0, 1, 0, 1]]),
-            torch.tensor([[0, 1, 0, 0], [0, 1, 0, 1]]),
-        ]
-        targets_list = [
-            torch.tensor([[0, 1, 1, 1], [0, 0, 0, 0], [0, 1, 0, 1]]),
-            torch.tensor([[0, 1, 0, 0], [0, 0, 1, 1], [1, 0, 1, 0]]),
-            torch.tensor([[0, 1, 0, 0], [0, 0, 1, 0]]),
-        ]
-
-        metric = metrics.MultilabelPrecisionRecallF1SupportMetric(
-            num_classes=num_classes, zero_division=zero_division
+        # @TODO: rewrite this block - should be without `num_classes`
+        if self.compute_per_class_metrics:
+            per_class_metrics = {
+                f"{metric_name}/class_{i:02d}": metric_value[i]
+                for metric_name, metric_value in zip(
+                    ("precision", "recall", "f1", "support"), per_class
+                )
+                for i in range(self.num_classes)
+            }
+            kv_metrics.update(per_class_metrics)
+        return kv_metrics
+
+    def update(self, outputs: torch.Tensor, targets: torch.Tensor) -> Tuple[Any, Any, Any, Any]:
+        """
+        Update statistics and return intermediate metrics results
+
+        Args:
+            outputs: prediction values
+            targets: true answers
+
+        Returns:
+            tuple of metrics intermediate results with per-class, micro, macro and
+                weighted averaging
+
+        """
+        tn, fp, fn, tp, support, num_classes = super().update(outputs=outputs, targets=targets)
+        per_class, micro, macro, weighted = get_aggregated_metrics(
+            tp=tp, fp=fp, fn=fn, support=support, zero_division=self.zero_division
         )
-        metric.reset()
-
-        for outputs, targets in zip(outputs_list, targets_list):
-            metric.update(outputs=outputs, targets=targets)
-
-        metric.compute()
-        # (
-        #     # per class precision, recall, f1, support
-        #     (
-        #         array([0.        , 0.66666667, 0.        , 0.4       ]),
-        #         array([0.        , 1.        , 0.        , 0.66666667]),
-        #         array([0.        , 0.7999952 , 0.        , 0.49999531]),
-        #         array([1., 4., 4., 3.])
-        #     ),
-        #     # micro precision, recall, f1, support
-        #     (0.46153846153846156, 0.5, 0.4799950080519163, None),
-        #     # macro precision, recall, f1, support
-        #     (0.2666666666666667, 0.4166666666666667, 0.32499762814318617, None),
-        #     # weighted precision, recall, f1, support
-        #     (0.32222222222222224, 0.5, 0.39166389481225283, None)
-        # )
-
-        metric.compute_key_value()
-        # {
-        #     'f1/_macro': 0.32499762814318617,
-        #     'f1/_micro': 0.4799950080519163,
-        #     'f1/_weighted': 0.39166389481225283,
-        #     'f1/class_00': 0.0,
-        #     'f1/class_01': 0.7999952000287999,
-        #     'f1/class_02': 0.0,
-        #     'f1/class_03': 0.49999531254394486,
-        #     'precision/_macro': 0.2666666666666667,
-        #     'precision/_micro': 0.46153846153846156,
-        #     'precision/_weighted': 0.32222222222222224,
-        #     'precision/class_00': 0.0,
-        #     'precision/class_01': 0.6666666666666667,
-        #     'precision/class_02': 0.0,
-        #     'precision/class_03': 0.4,
-        #     'recall/_macro': 0.4166666666666667,
-        #     'recall/_micro': 0.5,
-        #     'recall/_weighted': 0.5,
-        #     'recall/class_00': 0.0,
-        #     'recall/class_01': 1.0,
-        #     'recall/class_02': 0.0,
-        #     'recall/class_03': 0.6666666666666667,
-        #     'support/class_00': 1.0,
-        #     'support/class_01': 4.0,
-        #     'support/class_02': 4.0,
-        #     'support/class_03': 3.0
-        # }
-
-
-        metric.reset()
-        metric(outputs_list[0], targets_list[0])
-        # (
-        #     # per class precision, recall, f1, support
-        #     (
-        #         array([0., 1., 0., 1.]),
-        #         array([0. , 1. , 0. , 0.5]),
-        #         array([0.        , 0.999995  , 0.        , 0.66666222]),
-        #         array([0., 2., 1., 2.])
-        #     ),
-        #     # micro precision, recall, f1, support
-        #     (0.75, 0.6, 0.6666617284316411, None),
-        #     # macro precision, recall, f1, support
-        #     (0.5, 0.375, 0.41666430556921286, None),
-        #     # weighted precision, recall, f1, support
-        #     (0.8, 0.6000000000000001, 0.6666628889107407, None)
-        # )
+        if self.num_classes is None:
+            self.num_classes = num_classes
 
-    .. code-block:: python
+        return per_class, micro, macro, weighted
 
-        import torch
-        from torch.utils.data import DataLoader, TensorDataset
-        from catalyst import dl
+    def update_key_value(self, outputs: torch.Tensor, targets: torch.Tensor) -> Dict[str, float]:
+        """
+        Update statistics and return intermediate metrics results
 
-        # sample data
-        num_samples, num_features, num_classes = int(1e4), int(1e1), 4
-        X = torch.rand(num_samples, num_features)
-        y = (torch.rand(num_samples, num_classes) > 0.5).to(torch.float32)
+        Args:
+            outputs: prediction values
+            targets: true answers
 
-        # pytorch loaders
-        dataset = TensorDataset(X, y)
-        loader = DataLoader(dataset, batch_size=32, num_workers=1)
-        loaders = {"train": loader, "valid": loader}
+        Returns:
+            dict of metrics intermediate results
 
-        # model, criterion, optimizer, scheduler
-        model = torch.nn.Linear(num_features, num_classes)
-        criterion = torch.nn.BCEWithLogitsLoss()
-        optimizer = torch.optim.Adam(model.parameters())
-        scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [2])
+        """
+        per_class, micro, macro, weighted = self.update(outputs=outputs, targets=targets)
+        metrics = self._convert_metrics_to_kv(
+            per_class=per_class, micro=micro, macro=macro, weighted=weighted
+        )
+        return metrics
 
-        # model training
-        runner = dl.SupervisedRunner(
-            input_key="features", output_key="logits", target_key="targets", loss_key="loss"
+    def compute(self) -> Any:
+        """
+        Compute precision, recall, f1 score and support.
+        Compute micro, macro and weighted average for the metrics.
+
+        Returns:
+            list of aggregated metrics: per-class, micro, macro and weighted averaging of
+                precision, recall, f1 score and support metrics
+
+        """
+        # ddp hotfix, could be done better
+        # but metric must handle DDP on it's own
+        if self._ddp_backend == "xla":
+            device = get_device()
+            for key in self.statistics:
+                key_statistics = torch.tensor([self.statistics[key]], device=device)
+                key_statistics = xm.all_gather(key_statistics).sum(dim=0).cpu().numpy()
+                self.statistics[key] = key_statistics
+        elif self._ddp_backend == "ddp":
+            for key in self.statistics:
+                value: List[np.ndarray] = all_gather(self.statistics[key])
+                value: np.ndarray = np.sum(np.vstack(value), axis=0)
+                self.statistics[key] = value
+
+        per_class, micro, macro, weighted = get_aggregated_metrics(
+            tp=self.statistics["tp"],
+            fp=self.statistics["fp"],
+            fn=self.statistics["fn"],
+            support=self.statistics["support"],
+            zero_division=self.zero_division,
         )
-        runner.train(
-            model=model,
-            criterion=criterion,
-            optimizer=optimizer,
-            scheduler=scheduler,
-            loaders=loaders,
-            logdir="./logdir",
-            num_epochs=3,
-            valid_loader="valid",
-            valid_metric="accuracy",
-            minimize_valid_metric=False,
-            verbose=True,
-            callbacks=[
-                dl.BatchTransformCallback(
-                    transform=torch.sigmoid,
-                    scope="on_batch_end",
-                    input_key="logits",
-                    output_key="scores"
-                ),
-                dl.AUCCallback(input_key="scores", target_key="targets"),
-                dl.MultilabelAccuracyCallback(
-                    input_key="scores", target_key="targets", threshold=0.5
-                ),
-                dl.MultilabelPrecisionRecallF1SupportCallback(
-                    input_key="scores", target_key="targets", threshold=0.5
-                ),
-            ]
+        if self.compute_per_class_metrics:
+            return per_class, micro, macro, weighted
+        else:
+            return [], micro, macro, weighted
+
+    def compute_key_value(self) -> Dict[str, float]:
+        """
+        Compute precision, recall, f1 score and support.
+        Compute micro, macro and weighted average for the metrics.
+
+        Returns:
+            dict of metrics
+
+        """
+        per_class, micro, macro, weighted = self.compute()
+        metrics = self._convert_metrics_to_kv(
+            per_class=per_class, micro=micro, macro=macro, weighted=weighted
         )
+        return metrics
 
-    .. note::
-        Please follow the `minimal examples`_ sections for more use cases.
 
-        .. _`minimal examples`: https://github.com/catalyst-team/catalyst#minimal-examples
+class MultilabelPrecisionRecallF1SupportMetric(MultilabelStatisticsMetric):
+    """
+    Metric that can collect statistics and count precision, recall, f1_score and support with it.
+
+    Args:
+        zero_division: value to set in case of zero division during metrics
+            (precision, recall) computation; should be one of 0 or 1
+        compute_on_call: if True, allows compute metric's value on call
+        compute_per_class_metrics: boolean flag to compute per-class metrics
+            (default: SETTINGS.compute_per_class_metrics or False).
+        prefix: metrics prefix
+        suffix: metrics suffix
+        num_classes: number of classes
+
     """
 
     def __init__(
         self,
-        num_classes: int = None,
         zero_division: int = 0,
         compute_on_call: bool = True,
         compute_per_class_metrics: bool = SETTINGS.compute_per_class_metrics,
-        prefix: Optional[str] = None,
-        suffix: Optional[str] = None,
-    ):
-        """Init MultiLabelPrecisionRecallF1SupportMetric instance"""
+        prefix: str = None,
+        suffix: str = None,
+        num_classes: Optional[int] = None,
+    ) -> None:
+        """Init PrecisionRecallF1SupportMetric instance"""
         super().__init__(
-            compute_on_call=compute_on_call,
-            compute_per_class_metrics=compute_per_class_metrics,
-            prefix=prefix,
-            suffix=suffix,
-            num_classes=num_classes,
-            zero_division=zero_division,
-            mode="multilabel",
+            compute_on_call=compute_on_call, prefix=prefix, suffix=suffix, num_classes=num_classes
+        )
+        self.compute_per_class_metrics = compute_per_class_metrics
+        self.zero_division = zero_division
+        self.num_classes = num_classes
+        self.reset()
+
+    def _convert_metrics_to_kv(self, per_class, micro, macro, weighted) -> Dict[str, float]:
+        """
+        Convert metrics aggregation to key-value format
+
+        Args:
+            per_class: per-class metrics, array of shape (4, self.num_classes)
+                of precision, recall, f1 and support metrics
+            micro: micro averaged metrics, array of shape (self.num_classes)
+                of precision, recall, f1 and support metrics
+            macro: macro averaged metrics, array of shape (self.num_classes)
+                of precision, recall, f1 and support metrics
+            weighted: weighted averaged metrics, array of shape (self.num_classes)
+                of precision, recall, f1 and support metrics
+
+        Returns:
+            dict of key-value metrics
+
+        """
+        kv_metrics = {}
+        for aggregation_name, aggregated_metrics in zip(
+            ("_micro", "_macro", "_weighted"), (micro, macro, weighted)
+        ):
+            metrics = {
+                f"{metric_name}/{aggregation_name}": metric_value
+                for metric_name, metric_value in zip(
+                    ("precision", "recall", "f1"), aggregated_metrics[:-1]
+                )
+            }
+            kv_metrics.update(metrics)
+
+        # @TODO: rewrite this block - should be without `num_classes`
+        if self.compute_per_class_metrics:
+            per_class_metrics = {
+                f"{metric_name}/class_{i:02d}": metric_value[i]
+                for metric_name, metric_value in zip(
+                    ("precision", "recall", "f1", "support"), per_class
+                )
+                for i in range(self.num_classes)
+            }
+            kv_metrics.update(per_class_metrics)
+        return kv_metrics
+
+    def update(self, outputs: torch.Tensor, targets: torch.Tensor) -> Tuple[Any, Any, Any, Any]:
+        """
+        Update statistics and return intermediate metrics results
+
+        Args:
+            outputs: prediction values
+            targets: true answers
+
+        Returns:
+            tuple of metrics intermediate results with per-class, micro, macro and
+                weighted averaging
+
+        """
+        tn, fp, fn, tp, support, num_classes = super().update(outputs=outputs, targets=targets)
+        per_class, micro, macro, weighted = get_aggregated_metrics(
+            tp=tp, fp=fp, fn=fn, support=support, zero_division=self.zero_division
+        )
+        if self.num_classes is None:
+            self.num_classes = num_classes
+
+        return per_class, micro, macro, weighted
+
+    def update_key_value(self, outputs: torch.Tensor, targets: torch.Tensor) -> Dict[str, float]:
+        """
+        Update statistics and return intermediate metrics results
+
+        Args:
+            outputs: prediction values
+            targets: true answers
+
+        Returns:
+            dict of metrics intermediate results
+
+        """
+        per_class, micro, macro, weighted = self.update(outputs=outputs, targets=targets)
+        metrics = self._convert_metrics_to_kv(
+            per_class=per_class, micro=micro, macro=macro, weighted=weighted
         )
+        return metrics
+
+    def compute(self) -> Any:
+        """
+        Compute precision, recall, f1 score and support.
+        Compute micro, macro and weighted average for the metrics.
+
+        Returns:
+            list of aggregated metrics: per-class, micro, macro and weighted averaging of
+                precision, recall, f1 score and support metrics
+
+        """
+        # ddp hotfix, could be done better
+        # but metric must handle DDP on it's own
+        if self._ddp_backend == "xla":
+            device = get_device()
+            for key in self.statistics:
+                key_statistics = torch.tensor([self.statistics[key]], device=device)
+                key_statistics = xm.all_gather(key_statistics).sum(dim=0).cpu().numpy()
+                self.statistics[key] = key_statistics
+        elif self._ddp_backend == "ddp":
+            for key in self.statistics:
+                value: List[np.ndarray] = all_gather(self.statistics[key])
+                value: np.ndarray = np.sum(np.vstack(value), axis=0)
+                self.statistics[key] = value
+
+        per_class, micro, macro, weighted = get_aggregated_metrics(
+            tp=self.statistics["tp"],
+            fp=self.statistics["fp"],
+            fn=self.statistics["fn"],
+            support=self.statistics["support"],
+            zero_division=self.zero_division,
+        )
+        if self.compute_per_class_metrics:
+            return per_class, micro, macro, weighted
+        else:
+            return [], micro, macro, weighted
+
+    def compute_key_value(self) -> Dict[str, float]:
+        """
+        Compute precision, recall, f1 score and support.
+        Compute micro, macro and weighted average for the metrics.
+
+        Returns:
+            dict of metrics
+
+        """
+        per_class, micro, macro, weighted = self.compute()
+        metrics = self._convert_metrics_to_kv(
+            per_class=per_class, micro=micro, macro=macro, weighted=weighted
+        )
+        return metrics
 
 
 __all__ = [
diff --git a/catalyst/metrics/functional/_accuracy.py b/catalyst/metrics/functional/_accuracy.py
index 3dd5c10324..23620b9a08 100644
--- a/catalyst/metrics/functional/_accuracy.py
+++ b/catalyst/metrics/functional/_accuracy.py
@@ -174,7 +174,7 @@ def multilabel_accuracy(
         )
         # tensor(1.0)
     """
-    outputs, targets, _ = process_multilabel_components(outputs=outputs, targets=targets)
+    outputs, targets, _, _ = process_multilabel_components(outputs=outputs, targets=targets)
 
     outputs = (outputs > threshold).long()
     output = (targets.long() == outputs.long()).sum().float() / np.prod(targets.shape)
diff --git a/catalyst/metrics/functional/_average_precision.py b/catalyst/metrics/functional/_average_precision.py
index b052069bcd..93ef778530 100644
--- a/catalyst/metrics/functional/_average_precision.py
+++ b/catalyst/metrics/functional/_average_precision.py
@@ -40,7 +40,7 @@ def binary_average_precision(
     """
     # outputs - [bs; num_classes] with scores
     # targets - [bs; num_classes] with binary labels
-    outputs, targets, weights = process_multilabel_components(
+    outputs, targets, weights, _ = process_multilabel_components(
         outputs=outputs, targets=targets, weights=weights
     )
     if outputs.numel() == 0:
diff --git a/catalyst/metrics/functional/_classification.py b/catalyst/metrics/functional/_classification.py
index 8e0f423aad..d8db9a4b25 100644
--- a/catalyst/metrics/functional/_classification.py
+++ b/catalyst/metrics/functional/_classification.py
@@ -128,7 +128,7 @@ def precision_recall_fbeta_support(
         #     tensor([4., 4.]),          # per class support
         # )
     """
-    tn, fp, fn, tp, support = get_multiclass_statistics(
+    tn, fp, fn, tp, support, _ = get_multiclass_statistics(
         outputs=outputs, targets=targets, argmax_dim=argmax_dim, num_classes=num_classes
     )
     # @TODO: sync between metrics
diff --git a/catalyst/metrics/functional/_misc.py b/catalyst/metrics/functional/_misc.py
index 5e76eb081e..c456917b9a 100644
--- a/catalyst/metrics/functional/_misc.py
+++ b/catalyst/metrics/functional/_misc.py
@@ -102,7 +102,7 @@ def process_recsys_components(outputs: torch.Tensor, targets: torch.Tensor) -> t
 
 def process_multilabel_components(
     outputs: torch.Tensor, targets: torch.Tensor, weights: Optional[torch.Tensor] = None
-) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
     """General preprocessing for multilabel-based metrics.
 
     Args:
@@ -142,11 +142,13 @@ def process_multilabel_components(
             targets = F.one_hot(targets, num_classes).float()
         else:
             # binary case
+            num_classes = 2
             targets = targets.view(-1, 1)
     else:
         assert targets.dim() == 2, (
             "wrong `targets` size " "(should be 1D or 2D with one column per class)"
         )
+        num_classes = outputs.shape[-1]
 
     if weights is not None:
         assert weights.dim() == 1, "Weights dimension should be 1"
@@ -157,7 +159,7 @@ def process_multilabel_components(
 
     assert torch.equal(targets ** 2, targets), "targets should be binary (0 or 1)"
 
-    return outputs, targets, weights
+    return outputs, targets, weights, num_classes
 
 
 def get_binary_statistics(
@@ -200,7 +202,7 @@ def get_binary_statistics(
 
 def get_multiclass_statistics(
     outputs: Tensor, targets: Tensor, argmax_dim: int = -1, num_classes: Optional[int] = None
-) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor]:
+) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, int]:
     """
     Computes the number of true negative, false positive,
     false negative, true positive and support
@@ -254,12 +256,12 @@ def get_multiclass_statistics(
             support[class_index],
         ) = get_binary_statistics(outputs=outputs, targets=targets, label=class_index)
 
-    return tn, fp, fn, tp, support
+    return tn, fp, fn, tp, support, num_classes
 
 
 def get_multilabel_statistics(
     outputs: Tensor, targets: Tensor
-) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor]:
+) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, int]:
     """
     Computes the number of true negative, false positive,
     false negative, true positive and support
@@ -322,7 +324,9 @@ def get_multilabel_statistics(
         # )
 
     """
-    outputs, targets, _ = process_multilabel_components(outputs=outputs, targets=targets)
+    outputs, targets, _, num_classes = process_multilabel_components(
+        outputs=outputs, targets=targets
+    )
     assert outputs.shape == targets.shape
     num_classes = outputs.shape[-1]
 
@@ -343,7 +347,7 @@ def get_multilabel_statistics(
             support[class_index],
         ) = get_binary_statistics(outputs=class_outputs, targets=class_targets, label=1)
 
-    return tn, fp, fn, tp, support
+    return tn, fp, fn, tp, support, num_classes
 
 
 def get_default_topk_args(num_classes: int) -> Sequence[int]:


Test patch:
diff --git a/tests/catalyst/metrics/functional/test_misc.py b/tests/catalyst/metrics/functional/test_misc.py
index 44bb765795..f2e6272d1e 100644
--- a/tests/catalyst/metrics/functional/test_misc.py
+++ b/tests/catalyst/metrics/functional/test_misc.py
@@ -79,7 +79,7 @@ def test_get_binary_statistics(outputs, targets, tn_true, fp_true, fn_true, tp_t
 def test_get_multiclass_statistics(
     outputs, targets, tn_true, fp_true, fn_true, tp_true, support_true
 ):
-    tn, fp, fn, tp, support = get_multiclass_statistics(outputs, targets)
+    tn, fp, fn, tp, support, _ = get_multiclass_statistics(outputs, targets)
 
     assert torch.allclose(torch.tensor(tn_true).to(tn), tn)
     assert torch.allclose(torch.tensor(fp_true).to(fp), fp)
@@ -123,7 +123,7 @@ def test_get_multiclass_statistics(
 def test_get_multilabel_statistics(
     outputs, targets, tn_true, fp_true, fn_true, tp_true, support_true
 ):
-    tn, fp, fn, tp, support = get_multilabel_statistics(outputs, targets)
+    tn, fp, fn, tp, support, _ = get_multilabel_statistics(outputs, targets)
 
     assert torch.allclose(torch.tensor(tn_true).to(tn), tn)
     assert torch.allclose(torch.tensor(fp_true).to(fp), fp)
diff --git a/tests/catalyst/metrics/test_classification.py b/tests/catalyst/metrics/test_classification.py
index ee0ac7a340..550f2b60b9 100644
--- a/tests/catalyst/metrics/test_classification.py
+++ b/tests/catalyst/metrics/test_classification.py
@@ -259,13 +259,460 @@ def test_multiclass_metrics(
         assert abs(metrics[key] - true_values[key]) < EPS
 
 
+@pytest.mark.parametrize(
+    "outputs,targets,zero_division,true_values",
+    (
+        (
+            torch.tensor([1, 0, 3, 2, 2, 2]),
+            torch.tensor([0, 0, 2, 4, 3, 1]),
+            1,
+            {
+                "precision/_macro": 0.4,
+                "precision/_micro": 0.166667,
+                "precision/_weighted": 0.5,
+                "precision/class_00": 1.0,
+                "precision/class_01": 0.0,
+                "precision/class_02": 0.0,
+                "precision/class_03": 0.0,
+                "precision/class_04": 1.0,
+                "recall/_macro": 0.1,
+                "recall/_micro": 0.166667,
+                "recall/_weighted": 0.166667,
+                "recall/class_00": 0.5,
+                "recall/class_01": 0.0,
+                "recall/class_02": 0.0,
+                "recall/class_03": 0.0,
+                "recall/class_04": 0.0,
+                "f1/_macro": 0.133333,
+                "f1/_micro": 0.166667,
+                "f1/_weighted": 0.222222,
+                "f1/class_00": 0.666667,
+                "f1/class_01": 0.0,
+                "f1/class_02": 0.0,
+                "f1/class_03": 0.0,
+                "f1/class_04": 0.0,
+                "support/class_00": 2,
+                "support/class_01": 1,
+                "support/class_02": 1,
+                "support/class_03": 1,
+                "support/class_04": 1,
+            },
+        ),
+        (
+            torch.tensor([2, 2, 2, 3, 3, 4]),
+            torch.tensor([4, 4, 2, 1, 3, 1]),
+            0,
+            {
+                "precision/_macro": 0.166667,
+                "precision/_micro": 0.333333,
+                "precision/_weighted": 0.138889,
+                "precision/class_00": 0.0,
+                "precision/class_01": 0.0,
+                "precision/class_02": 0.333333,
+                "precision/class_03": 0.5,
+                "precision/class_04": 0.0,
+                "recall/_macro": 0.4,
+                "recall/_micro": 0.333333,
+                "recall/_weighted": 0.333333,
+                "recall/class_00": 0.0,
+                "recall/class_01": 0.0,
+                "recall/class_02": 1.0,
+                "recall/class_03": 1.0,
+                "recall/class_04": 0.0,
+                "f1/_macro": 0.233333,
+                "f1/_micro": 0.333333,
+                "f1/_weighted": 0.194444,
+                "f1/class_00": 0.0,
+                "f1/class_01": 0.0,
+                "f1/class_02": 0.5,
+                "f1/class_03": 0.666667,
+                "f1/class_04": 0.0,
+                "support/class_00": 0,
+                "support/class_01": 2,
+                "support/class_02": 1,
+                "support/class_03": 1,
+                "support/class_04": 2,
+            },
+        ),
+        (
+            torch.tensor([2, 2, 2, 3, 3, 4]),
+            torch.tensor([4, 4, 2, 1, 3, 1]),
+            1,
+            {
+                "precision/_macro": 0.566667,
+                "precision/_micro": 0.333333,
+                "precision/_weighted": 0.472222,
+                "precision/class_00": 1.0,
+                "precision/class_01": 1.0,
+                "precision/class_02": 0.333333,
+                "precision/class_03": 0.5,
+                "precision/class_04": 0.0,
+                "recall/_macro": 0.6,
+                "recall/_micro": 0.333333,
+                "recall/_weighted": 0.333333,
+                "recall/class_00": 1.0,
+                "recall/class_01": 0.0,
+                "recall/class_02": 1.0,
+                "recall/class_03": 1.0,
+                "recall/class_04": 0.0,
+                "f1/_macro": 0.433333,
+                "f1/_micro": 0.333333,
+                "f1/_weighted": 0.194444,
+                "f1/class_00": 1.0,
+                "f1/class_01": 0.0,
+                "f1/class_02": 0.5,
+                "f1/class_03": 0.666667,
+                "f1/class_04": 0.0,
+                "support/class_00": 0,
+                "support/class_01": 2,
+                "support/class_02": 1,
+                "support/class_03": 1,
+                "support/class_04": 2,
+            },
+        ),
+        (
+            torch.tensor([5, 1, 4, 0, 4, 6, 2, 2, 0, 5]),
+            torch.tensor([1, 2, 1, 1, 2, 5, 2, 0, 6, 6]),
+            1,
+            {
+                "precision/_macro": 0.214286,
+                "precision/_micro": 0.1,
+                "precision/_weighted": 0.15,
+                "precision/class_00": 0.0,
+                "precision/class_01": 0.0,
+                "precision/class_02": 0.5,
+                "precision/class_03": 1.0,
+                "precision/class_04": 0.0,
+                "precision/class_05": 0.0,
+                "precision/class_06": 0.0,
+                "recall/_macro": 0.333333,
+                "recall/_micro": 0.1,
+                "recall/_weighted": 0.1,
+                "recall/class_00": 0.0,
+                "recall/class_01": 0.0,
+                "recall/class_02": 0.333333,
+                "recall/class_03": 1.0,
+                "recall/class_04": 1.0,
+                "recall/class_05": 0.0,
+                "recall/class_06": 0.0,
+                "f1/_macro": 0.2,
+                "f1/_micro": 0.1,
+                "f1/_weighted": 0.12,
+                "f1/class_00": 0.0,
+                "f1/class_01": 0.0,
+                "f1/class_02": 0.4,
+                "f1/class_03": 1.0,
+                "f1/class_04": 0.0,
+                "f1/class_05": 0.0,
+                "f1/class_06": 0.0,
+                "support/class_00": 1,
+                "support/class_01": 3,
+                "support/class_02": 3,
+                "support/class_03": 0,
+                "support/class_04": 0,
+                "support/class_05": 1,
+                "support/class_06": 2,
+            },
+        ),
+        (
+            torch.tensor([2, 2, 1, 1, 2, 2, 1, 2, 2, 0]),
+            torch.tensor([1, 2, 0, 2, 2, 1, 1, 2, 0, 2]),
+            0,
+            {
+                "precision/_macro": 0.277778,
+                "precision/_micro": 0.4,
+                "precision/_weighted": 0.35,
+                "precision/class_00": 0.0,
+                "precision/class_01": 0.333333,
+                "precision/class_02": 0.5,
+                "recall/_macro": 0.311111,
+                "recall/_micro": 0.4,
+                "recall/_weighted": 0.4,
+                "recall/class_00": 0.0,
+                "recall/class_01": 0.333333,
+                "recall/class_02": 0.6,
+                "f1/_macro": 0.292929,
+                "f1/_micro": 0.4,
+                "f1/_weighted": 0.372727,
+                "f1/class_00": 0.0,
+                "f1/class_01": 0.333333,
+                "f1/class_02": 0.545455,
+                "support/class_00": 2,
+                "support/class_01": 3,
+                "support/class_02": 5,
+            },
+        ),
+        (
+            torch.tensor([2, 0, 0, 0]),
+            torch.tensor([2, 2, 0, 2]),
+            0,
+            {
+                "precision/_macro": 0.444444,
+                "precision/_micro": 0.5,
+                "precision/_weighted": 0.833333,
+                "precision/class_00": 0.333333,
+                "precision/class_01": 0.0,
+                "precision/class_02": 1.0,
+                "recall/_macro": 0.444444,
+                "recall/_micro": 0.5,
+                "recall/_weighted": 0.5,
+                "recall/class_00": 1.0,
+                "recall/class_01": 0.0,
+                "recall/class_02": 0.333333,
+                "f1/_macro": 0.333333,
+                "f1/_micro": 0.5,
+                "f1/_weighted": 0.5,
+                "f1/class_00": 0.5,
+                "f1/class_01": 0.0,
+                "f1/class_02": 0.5,
+                "support/class_00": 1,
+                "support/class_01": 0,
+                "support/class_02": 3,
+            },
+        ),
+    ),
+)
+def test_multiclass_metrics_without_num_classes(
+    outputs: torch.Tensor,
+    targets: torch.Tensor,
+    zero_division: int,
+    true_values: Dict[str, float],
+) -> None:
+    """
+    Test multiclass metric
+
+    Args:
+        outputs: tensor of predictions
+        targets: tensor of targets
+        zero_division: zero division policy flag
+        true_values: true values of metrics
+    """
+    metric = MulticlassPrecisionRecallF1SupportMetric(zero_division=zero_division)
+    metric.update(outputs=outputs, targets=targets)
+    metrics = metric.compute_key_value()
+    for key in true_values:
+        assert key in metrics
+        assert abs(metrics[key] - true_values[key]) < EPS
+
+
 @pytest.mark.parametrize(
     "outputs,targets,num_classes,zero_division,true_values",
     (
         (
             torch.tensor([[0, 1, 0], [1, 1, 0], [0, 0, 1], [0, 0, 0], [0, 1, 1]]),
             torch.tensor([[0, 1, 1], [1, 1, 1], [0, 0, 0], [0, 0, 1], [0, 1, 1]]),
-            3,
+            3,
+            0,
+            {
+                "precision/_macro": 0.833333,
+                "precision/_micro": 0.833333,
+                "precision/_weighted": 0.75,
+                "precision/class_00": 1.0,
+                "precision/class_01": 1.0,
+                "precision/class_02": 0.5,
+                "recall/_macro": 0.75,
+                "recall/_micro": 0.625,
+                "recall/_weighted": 0.625,
+                "recall/class_00": 1.0,
+                "recall/class_01": 1.0,
+                "recall/class_02": 0.25,
+                "f1/_macro": 0.777778,
+                "f1/_micro": 0.714286,
+                "f1/_weighted": 0.666667,
+                "f1/class_00": 1.0,
+                "f1/class_01": 1.0,
+                "f1/class_02": 0.333333,
+                "support/class_00": 1,
+                "support/class_01": 3,
+                "support/class_02": 4,
+            },
+        ),
+        (
+            torch.tensor([[0, 1, 0, 0], [1, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 1, 1, 0]]),
+            torch.tensor([[0, 1, 1, 1], [1, 1, 1, 0], [0, 0, 0, 0], [0, 0, 1, 1], [0, 1, 1, 0]]),
+            4,
+            0,
+            {
+                "precision/_macro": 0.625,
+                "precision/_micro": 0.833333,
+                "precision/_weighted": 0.6,
+                "precision/class_00": 1.0,
+                "precision/class_01": 1.0,
+                "precision/class_02": 0.5,
+                "precision/class_03": 0.0,
+                "recall/_macro": 0.5625,
+                "recall/_micro": 0.5,
+                "recall/_weighted": 0.5,
+                "recall/class_00": 1.0,
+                "recall/class_01": 1.0,
+                "recall/class_02": 0.25,
+                "recall/class_03": 0.0,
+                "f1/_macro": 0.583333,
+                "f1/_micro": 0.625,
+                "f1/_weighted": 0.533333,
+                "f1/class_00": 1.0,
+                "f1/class_01": 1.0,
+                "f1/class_02": 0.333333,
+                "f1/class_03": 0.0,
+                "support/class_00": 1,
+                "support/class_01": 3,
+                "support/class_02": 4,
+                "support/class_03": 2,
+            },
+        ),
+        (
+            torch.tensor([[0, 1, 0, 0], [1, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 1, 1, 0]]),
+            torch.tensor([[0, 1, 1, 1], [1, 1, 1, 0], [0, 0, 0, 0], [0, 0, 1, 1], [0, 1, 1, 0]]),
+            4,
+            1,
+            {
+                "precision/_macro": 0.875,
+                "precision/_micro": 0.833333,
+                "precision/_weighted": 0.8,
+                "precision/class_00": 1.0,
+                "precision/class_01": 1.0,
+                "precision/class_02": 0.5,
+                "precision/class_03": 1.0,
+                "recall/_macro": 0.5625,
+                "recall/_micro": 0.5,
+                "recall/_weighted": 0.5,
+                "recall/class_00": 1.0,
+                "recall/class_01": 1.0,
+                "recall/class_02": 0.25,
+                "recall/class_03": 0.0,
+                "f1/_macro": 0.583333,
+                "f1/_micro": 0.625,
+                "f1/_weighted": 0.533333,
+                "f1/class_00": 1.0,
+                "f1/class_01": 1.0,
+                "f1/class_02": 0.333333,
+                "f1/class_03": 0.0,
+                "support/class_00": 1,
+                "support/class_01": 3,
+                "support/class_02": 4,
+                "support/class_03": 2,
+            },
+        ),
+        (
+            torch.tensor(
+                [
+                    [0, 1, 0, 0, 1],
+                    [1, 1, 0, 0, 1],
+                    [0, 0, 1, 0, 1],
+                    [0, 0, 0, 0, 0],
+                    [0, 1, 1, 0, 0],
+                    [0, 1, 0, 0, 1],
+                    [1, 1, 0, 0, 1],
+                    [0, 0, 1, 0, 1],
+                    [0, 1, 1, 0, 0],
+                ]
+            ),
+            torch.tensor(
+                [
+                    [0, 1, 1, 1, 0],
+                    [1, 1, 1, 0, 1],
+                    [0, 0, 0, 0, 0],
+                    [0, 0, 1, 1, 0],
+                    [0, 1, 1, 0, 0],
+                    [0, 1, 1, 1, 0],
+                    [1, 1, 1, 0, 1],
+                    [0, 0, 0, 0, 0],
+                    [0, 0, 1, 1, 0],
+                ]
+            ),
+            5,
+            0,
+            {
+                "precision/_macro": 0.533333,
+                "precision/_micro": 0.611111,
+                "precision/_weighted": 0.516667,
+                "precision/class_00": 1.0,
+                "precision/class_01": 0.833333,
+                "precision/class_02": 0.5,
+                "precision/class_03": 0.0,
+                "precision/class_04": 0.333333,
+                "recall/_macro": 0.657143,
+                "recall/_micro": 0.55,
+                "recall/_weighted": 0.55,
+                "recall/class_00": 1.0,
+                "recall/class_01": 1.0,
+                "recall/class_02": 0.285714,
+                "recall/class_03": 0.0,
+                "recall/class_04": 1.0,
+                "f1/_macro": 0.554545,
+                "f1/_micro": 0.578947,
+                "f1/_weighted": 0.504545,
+                "f1/class_00": 1.0,
+                "f1/class_01": 0.909091,
+                "f1/class_02": 0.363636,
+                "f1/class_03": 0.0,
+                "f1/class_04": 0.5,
+                "support/class_00": 2,
+                "support/class_01": 5,
+                "support/class_02": 7,
+                "support/class_03": 4,
+                "support/class_04": 2,
+            },
+        ),
+        (
+            torch.tensor([[0, 1], [1, 0], [0, 1], [0, 0], [1, 1]]),
+            torch.tensor([[1, 1], [1, 1], [0, 0], [0, 1], [1, 1]]),
+            2,
+            0,
+            {
+                "precision/_macro": 0.833333,
+                "precision/_micro": 0.8,
+                "precision/_weighted": 0.809524,
+                "precision/class_00": 1.0,
+                "precision/class_01": 0.666667,
+                "recall/_macro": 0.583333,
+                "recall/_micro": 0.571429,
+                "recall/_weighted": 0.571429,
+                "recall/class_00": 0.666667,
+                "recall/class_01": 0.5,
+                "f1/_macro": 0.685714,
+                "f1/_micro": 0.666667,
+                "f1/_weighted": 0.669388,
+                "f1/class_00": 0.8,
+                "f1/class_01": 0.571429,
+                "support/class_00": 3,
+                "support/class_01": 4,
+            },
+        ),
+    ),
+)
+def test_multilabel_metrics(
+    outputs: torch.Tensor,
+    targets: torch.Tensor,
+    num_classes: int,
+    zero_division: int,
+    true_values: Dict[str, float],
+) -> None:
+    """
+    Test multilabel metric
+
+    Args:
+        outputs: tensor of predictions
+        targets: tensor of targets
+        zero_division: zero division policy flag
+        true_values: true values of metrics
+    """
+    metric = MultilabelPrecisionRecallF1SupportMetric(
+        num_classes=num_classes, zero_division=zero_division
+    )
+    metric.update(outputs=outputs, targets=targets)
+    metrics = metric.compute_key_value()
+    for key in true_values:
+        assert key in metrics
+        assert abs(metrics[key] - true_values[key]) < EPS
+
+
+@pytest.mark.parametrize(
+    "outputs,targets,zero_division,true_values",
+    (
+        (
+            torch.tensor([[0, 1, 0], [1, 1, 0], [0, 0, 1], [0, 0, 0], [0, 1, 1]]),
+            torch.tensor([[0, 1, 1], [1, 1, 1], [0, 0, 0], [0, 0, 1], [0, 1, 1]]),
             0,
             {
                 "precision/_macro": 0.833333,
@@ -294,7 +741,6 @@ def test_multiclass_metrics(
         (
             torch.tensor([[0, 1, 0, 0], [1, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 1, 1, 0]]),
             torch.tensor([[0, 1, 1, 1], [1, 1, 1, 0], [0, 0, 0, 0], [0, 0, 1, 1], [0, 1, 1, 0]]),
-            4,
             0,
             {
                 "precision/_macro": 0.625,
@@ -327,7 +773,6 @@ def test_multiclass_metrics(
         (
             torch.tensor([[0, 1, 0, 0], [1, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 1, 1, 0]]),
             torch.tensor([[0, 1, 1, 1], [1, 1, 1, 0], [0, 0, 0, 0], [0, 0, 1, 1], [0, 1, 1, 0]]),
-            4,
             1,
             {
                 "precision/_macro": 0.875,
@@ -384,7 +829,6 @@ def test_multiclass_metrics(
                     [0, 0, 1, 1, 0],
                 ]
             ),
-            5,
             0,
             {
                 "precision/_macro": 0.533333,
@@ -421,7 +865,6 @@ def test_multiclass_metrics(
         (
             torch.tensor([[0, 1], [1, 0], [0, 1], [0, 0], [1, 1]]),
             torch.tensor([[1, 1], [1, 1], [0, 0], [0, 1], [1, 1]]),
-            2,
             0,
             {
                 "precision/_macro": 0.833333,
@@ -445,10 +888,9 @@ def test_multiclass_metrics(
         ),
     ),
 )
-def test_multilabel_metrics(
+def test_multilabel_metrics_without_num_classes(
     outputs: torch.Tensor,
     targets: torch.Tensor,
-    num_classes: int,
     zero_division: int,
     true_values: Dict[str, float],
 ) -> None:
@@ -461,9 +903,7 @@ def test_multilabel_metrics(
         zero_division: zero division policy flag
         true_values: true values of metrics
     """
-    metric = MultilabelPrecisionRecallF1SupportMetric(
-        num_classes=num_classes, zero_division=zero_division
-    )
+    metric = MultilabelPrecisionRecallF1SupportMetric(zero_division=zero_division)
     metric.update(outputs=outputs, targets=targets)
     metrics = metric.compute_key_value()
     for key in true_values:
@@ -671,54 +1111,227 @@ def test_update(
                     "f1/_weighted": 0.666667,
                     "precision/class_00": 1.0,
                     "precision/class_01": 1.0,
-                    "precision/class_02": 0.0,
+                    "precision/class_02": 0.0,
+                    "precision/class_03": 0.0,
+                    "recall/class_00": 1.0,
+                    "recall/class_01": 1.0,
+                    "recall/class_02": 0.0,
+                    "recall/class_03": 0.0,
+                    "f1/class_00": 1.0,
+                    "f1/class_01": 1.0,
+                    "f1/class_02": 0.0,
+                    "f1/class_03": 0.0,
+                    "support/class_00": 1,
+                    "support/class_01": 1,
+                    "support/class_02": 1,
+                    "support/class_03": 0,
+                },
+            ],
+            {
+                "precision/_micro": 0.75,
+                "recall/_micro": 0.75,
+                "f1/_micro": 0.75,
+                "precision/_macro": 0.75,
+                "recall/_macro": 0.791667,
+                "f1/_macro": 0.741667,
+                "precision/_weighted": 0.8125,
+                "recall/_weighted": 0.75,
+                "f1/_weighted": 0.758333,
+                "precision/class_00": 1.0,
+                "precision/class_01": 1.0,
+                "precision/class_02": 0.5,
+                "precision/class_03": 0.5,
+                "recall/class_00": 1.0,
+                "recall/class_01": 0.666667,
+                "recall/class_02": 0.5,
+                "recall/class_03": 1.0,
+                "f1/class_00": 1.0,
+                "f1/class_01": 0.8,
+                "f1/class_02": 0.5,
+                "f1/class_03": 0.666667,
+                "support/class_00": 2,
+                "support/class_01": 3,
+                "support/class_02": 2,
+                "support/class_03": 1,
+            },
+        ),
+        (
+            [torch.tensor([0, 1, 2, 4]), torch.tensor([2, 3, 3, 2]), torch.tensor([0, 1, 3, 4])],
+            [torch.tensor([0, 1, 1, 4]), torch.tensor([2, 3, 3, 4]), torch.tensor([0, 1, 2, 4])],
+            5,
+            1,
+            [
+                {
+                    "precision/_micro": 0.75,
+                    "recall/_micro": 0.75,
+                    "f1/_micro": 0.75,
+                    "precision/_macro": 0.8,
+                    "recall/_macro": 0.9,
+                    "f1/_macro": 0.733333,
+                    "precision/_weighted": 1.0,
+                    "recall/_weighted": 0.75,
+                    "f1/_weighted": 0.833333,
+                    "precision/class_00": 1.0,
+                    "precision/class_01": 1.0,
+                    "precision/class_02": 0.0,
+                    "precision/class_03": 1.0,
+                    "precision/class_04": 1.0,
+                    "recall/class_00": 1.0,
+                    "recall/class_01": 0.5,
+                    "recall/class_02": 1.0,
+                    "recall/class_03": 1.0,
+                    "recall/class_04": 1.0,
+                    "f1/class_00": 1.0,
+                    "f1/class_01": 0.666667,
+                    "f1/class_02": 0.0,
+                    "f1/class_03": 1.0,
+                    "f1/class_04": 1.0,
+                    "support/class_00": 1,
+                    "support/class_01": 2,
+                    "support/class_02": 0,
+                    "support/class_03": 0,
+                    "support/class_04": 1,
+                },
+                {
+                    "precision/_micro": 0.75,
+                    "recall/_micro": 0.75,
+                    "f1/_micro": 0.75,
+                    "precision/_macro": 0.9,
+                    "recall/_macro": 0.8,
+                    "f1/_macro": 0.733333,
+                    "precision/_weighted": 0.875,
+                    "recall/_weighted": 0.75,
+                    "f1/_weighted": 0.666667,
+                    "precision/class_00": 1.0,
+                    "precision/class_01": 1.0,
+                    "precision/class_02": 0.5,
+                    "precision/class_03": 1.0,
+                    "precision/class_04": 1.0,
+                    "recall/class_00": 1.0,
+                    "recall/class_01": 1.0,
+                    "recall/class_02": 1.0,
+                    "recall/class_03": 1.0,
+                    "recall/class_04": 0.0,
+                    "f1/class_00": 1.0,
+                    "f1/class_01": 1.0,
+                    "f1/class_02": 0.666667,
+                    "f1/class_03": 1.0,
+                    "f1/class_04": 0.0,
+                    "support/class_00": 0,
+                    "support/class_01": 0,
+                    "support/class_02": 1,
+                    "support/class_03": 2,
+                    "support/class_04": 1,
+                },
+                {
+                    "precision/_micro": 0.75,
+                    "recall/_micro": 0.75,
+                    "f1/_micro": 0.75,
+                    "precision/_macro": 0.8,
+                    "recall/_macro": 0.8,
+                    "f1/_macro": 0.6,
+                    "precision/_weighted": 1.0,
+                    "recall/_weighted": 0.75,
+                    "f1/_weighted": 0.75,
+                    "precision/class_00": 1.0,
+                    "precision/class_01": 1.0,
+                    "precision/class_02": 1.0,
                     "precision/class_03": 0.0,
+                    "precision/class_04": 1.0,
                     "recall/class_00": 1.0,
                     "recall/class_01": 1.0,
                     "recall/class_02": 0.0,
-                    "recall/class_03": 0.0,
+                    "recall/class_03": 1.0,
+                    "recall/class_04": 1.0,
                     "f1/class_00": 1.0,
                     "f1/class_01": 1.0,
                     "f1/class_02": 0.0,
                     "f1/class_03": 0.0,
+                    "f1/class_04": 1.0,
                     "support/class_00": 1,
                     "support/class_01": 1,
                     "support/class_02": 1,
                     "support/class_03": 0,
+                    "support/class_04": 1,
                 },
             ],
             {
                 "precision/_micro": 0.75,
                 "recall/_micro": 0.75,
                 "f1/_micro": 0.75,
-                "precision/_macro": 0.75,
-                "recall/_macro": 0.791667,
-                "f1/_macro": 0.741667,
-                "precision/_weighted": 0.8125,
+                "precision/_macro": 0.8,
+                "recall/_macro": 0.766667,
+                "f1/_macro": 0.76,
+                "precision/_weighted": 0.833333,
                 "recall/_weighted": 0.75,
-                "f1/_weighted": 0.758333,
+                "f1/_weighted": 0.766667,
                 "precision/class_00": 1.0,
                 "precision/class_01": 1.0,
-                "precision/class_02": 0.5,
-                "precision/class_03": 0.5,
+                "precision/class_02": 0.333333,
+                "precision/class_03": 0.666667,
+                "precision/class_04": 1.0,
                 "recall/class_00": 1.0,
                 "recall/class_01": 0.666667,
                 "recall/class_02": 0.5,
                 "recall/class_03": 1.0,
+                "recall/class_04": 0.666667,
                 "f1/class_00": 1.0,
                 "f1/class_01": 0.8,
-                "f1/class_02": 0.5,
-                "f1/class_03": 0.666667,
+                "f1/class_02": 0.4,
+                "f1/class_03": 0.8,
+                "f1/class_04": 0.8,
                 "support/class_00": 2,
                 "support/class_01": 3,
                 "support/class_02": 2,
-                "support/class_03": 1,
+                "support/class_03": 2,
+                "support/class_04": 3,
             },
         ),
+    ),
+)
+def test_update_key_value_multiclass(
+    outputs_list: Iterable[torch.Tensor],
+    targets_list: Iterable[torch.Tensor],
+    num_classes: int,
+    zero_division: int,
+    update_true_values: Iterable[Dict[str, float]],
+    compute_true_value: Dict[str, float],
+) -> None:
+    """
+    This test checks that metrics update works correctly with multiple calls.
+    Metric should update statistics and return metrics for tmp input, so in this test
+    we call update_key_value multiple times and check that all the intermediate metrics values
+    are correct. After all the updates it checks that metrics computed with accumulated
+    statistics are correct too.
+
+    Args:
+        outputs_list: sequence of predictions
+        targets_list: sequence of targets
+        num_classes: number of classes
+        zero_division: int value, should be 0 or 1; return it in metrics in case of zero division
+        update_true_values: sequence of true intermediate metrics
+        compute_true_value: total metrics value for all the items from output_list and targets_list
+    """
+    metric = MulticlassPrecisionRecallF1SupportMetric(
+        num_classes=num_classes, zero_division=zero_division
+    )
+    for outputs, targets, update_true_value in zip(outputs_list, targets_list, update_true_values):
+        intermediate_metrics = metric.update_key_value(outputs=outputs, targets=targets)
+        for key in update_true_value:
+            assert key in intermediate_metrics
+            assert abs(intermediate_metrics[key] - update_true_value[key]) < EPS
+    metrics = metric.compute_key_value()
+    for key in compute_true_value:
+        assert key in metrics
+        assert abs(metrics[key] - compute_true_value[key]) < EPS
+
+
+@pytest.mark.parametrize(
+    "outputs_list,targets_list,zero_division,update_true_values,compute_true_value",
+    (
         (
             [torch.tensor([0, 1, 2, 4]), torch.tensor([2, 3, 3, 2]), torch.tensor([0, 1, 3, 4])],
             [torch.tensor([0, 1, 1, 4]), torch.tensor([2, 3, 3, 4]), torch.tensor([0, 1, 2, 4])],
-            5,
             1,
             [
                 {
@@ -849,10 +1462,9 @@ def test_update(
         ),
     ),
 )
-def test_update_key_value_multiclass(
+def test_update_key_value_multiclass_without_num_classes(
     outputs_list: Iterable[torch.Tensor],
     targets_list: Iterable[torch.Tensor],
-    num_classes: int,
     zero_division: int,
     update_true_values: Iterable[Dict[str, float]],
     compute_true_value: Dict[str, float],
@@ -867,14 +1479,11 @@ def test_update_key_value_multiclass(
     Args:
         outputs_list: sequence of predictions
         targets_list: sequence of targets
-        num_classes: number of classes
         zero_division: int value, should be 0 or 1; return it in metrics in case of zero division
         update_true_values: sequence of true intermediate metrics
         compute_true_value: total metrics value for all the items from output_list and targets_list
     """
-    metric = MulticlassPrecisionRecallF1SupportMetric(
-        num_classes=num_classes, zero_division=zero_division
-    )
+    metric = MulticlassPrecisionRecallF1SupportMetric(zero_division=zero_division)
     for outputs, targets, update_true_value in zip(outputs_list, targets_list, update_true_values):
         intermediate_metrics = metric.update_key_value(outputs=outputs, targets=targets)
         for key in update_true_value:
@@ -1052,6 +1661,167 @@ def test_update_key_value_multilabel(
         assert abs(metrics[key] - compute_true_value[key]) < EPS
 
 
+@pytest.mark.parametrize(
+    "outputs_list,targets_list,zero_division,update_true_values,compute_true_value",
+    (
+        (
+            [
+                torch.tensor([[0, 1, 0, 1], [0, 0, 0, 0], [0, 1, 1, 0]]),
+                torch.tensor([[0, 1, 1, 1], [0, 0, 0, 1], [0, 1, 0, 1]]),
+                torch.tensor([[0, 1, 0, 0], [0, 1, 0, 1]]),
+            ],
+            [
+                torch.tensor([[0, 1, 1, 1], [0, 0, 0, 0], [0, 1, 0, 1]]),
+                torch.tensor([[0, 1, 0, 0], [0, 0, 1, 1], [1, 0, 1, 0]]),
+                torch.tensor([[0, 1, 0, 0], [0, 0, 1, 0]]),
+            ],
+            1,
+            [
+                {
+                    "precision/_micro": 0.75,
+                    "recall/_micro": 0.6,
+                    "f1/_micro": 0.666667,
+                    "precision/_macro": 0.75,
+                    "recall/_macro": 0.625,
+                    "f1/_macro": 0.666667,
+                    "precision/_weighted": 0.8,
+                    "recall/_weighted": 0.6,
+                    "f1/_weighted": 0.666667,
+                    "precision/class_00": 1.0,
+                    "precision/class_01": 1.0,
+                    "precision/class_02": 0.0,
+                    "precision/class_03": 1.0,
+                    "recall/class_00": 1.0,
+                    "recall/class_01": 1.0,
+                    "recall/class_02": 0.0,
+                    "recall/class_03": 0.5,
+                    "f1/class_00": 1.0,
+                    "f1/class_01": 1.0,
+                    "f1/class_02": 0.0,
+                    "f1/class_03": 0.666667,
+                    "support/class_00": 0,
+                    "support/class_01": 2,
+                    "support/class_02": 1,
+                    "support/class_03": 2,
+                },
+                {
+                    "precision/_micro": 0.333333,
+                    "recall/_micro": 0.4,
+                    "f1/_micro": 0.363636,
+                    "precision/_macro": 0.458333,
+                    "recall/_macro": 0.5,
+                    "f1/_macro": 0.291667,
+                    "precision/_weighted": 0.366667,
+                    "recall/_weighted": 0.4,
+                    "f1/_weighted": 0.233333,
+                    "precision/class_00": 1.0,
+                    "precision/class_01": 0.5,
+                    "precision/class_02": 0.0,
+                    "precision/class_03": 0.333333,
+                    "recall/class_00": 0.0,
+                    "recall/class_01": 1.0,
+                    "recall/class_02": 0.0,
+                    "recall/class_03": 1.0,
+                    "f1/class_00": 0.0,
+                    "f1/class_01": 0.666667,
+                    "f1/class_02": 0.0,
+                    "f1/class_03": 0.5,
+                    "support/class_00": 1,
+                    "support/class_01": 1,
+                    "support/class_02": 2,
+                    "support/class_03": 1,
+                },
+                {
+                    "precision/_micro": 0.333333,
+                    "recall/_micro": 0.5,
+                    "f1/_micro": 0.4,
+                    "precision/_macro": 0.625,
+                    "recall/_macro": 0.75,
+                    "f1/_macro": 0.416667,
+                    "precision/_weighted": 0.75,
+                    "recall/_weighted": 0.5,
+                    "f1/_weighted": 0.333333,
+                    "precision/class_00": 1.0,
+                    "precision/class_01": 0.5,
+                    "precision/class_02": 1.0,
+                    "precision/class_03": 0.0,
+                    "recall/class_00": 1.0,
+                    "recall/class_01": 1.0,
+                    "recall/class_02": 0.0,
+                    "recall/class_03": 1.0,
+                    "f1/class_00": 1.0,
+                    "f1/class_01": 0.666667,
+                    "f1/class_02": 0.0,
+                    "f1/class_03": 0.0,
+                    "support/class_00": 0,
+                    "support/class_01": 1,
+                    "support/class_02": 1,
+                    "support/class_03": 0,
+                },
+            ],
+            {
+                "precision/_micro": 0.461538,
+                "recall/_micro": 0.5,
+                "f1/_micro": 0.48,
+                "precision/_macro": 0.516667,
+                "recall/_macro": 0.416667,
+                "f1/_macro": 0.325,
+                "precision/_weighted": 0.405556,
+                "recall/_weighted": 0.5,
+                "f1/_weighted": 0.391667,
+                "precision/class_00": 1.0,
+                "precision/class_01": 0.666667,
+                "precision/class_02": 0.0,
+                "precision/class_03": 0.4,
+                "recall/class_00": 0.0,
+                "recall/class_01": 1.0,
+                "recall/class_02": 0.0,
+                "recall/class_03": 0.666667,
+                "f1/class_00": 0.0,
+                "f1/class_01": 0.8,
+                "f1/class_02": 0.0,
+                "f1/class_03": 0.5,
+                "support/class_00": 1,
+                "support/class_01": 4,
+                "support/class_02": 4,
+                "support/class_03": 3,
+            },
+        ),
+    ),
+)
+def test_update_key_value_multilabel_without_num_classes(
+    outputs_list: Iterable[torch.Tensor],
+    targets_list: Iterable[torch.Tensor],
+    zero_division: int,
+    update_true_values: Iterable[Dict[str, float]],
+    compute_true_value: Dict[str, float],
+):
+    """
+    This test checks that metrics update works correctly with multiple calls.
+    Metric should update statistics and return metrics for tmp input, so in this test
+    we call update_key_value multiple times and check that all the intermediate metrics values
+    are correct. After all the updates it checks that metrics computed with accumulated
+    statistics are correct too.
+
+    Args:
+        outputs_list: sequence of predictions
+        targets_list: sequence of targets
+        zero_division: int value, should be 0 or 1; return it in metrics in case of zero division
+        update_true_values: sequence of true intermediate metrics
+        compute_true_value: total metrics value for all the items from output_list and targets_list
+    """
+    metric = MultilabelPrecisionRecallF1SupportMetric(zero_division=zero_division)
+    for outputs, targets, update_true_value in zip(outputs_list, targets_list, update_true_values):
+        intermediate_metrics = metric.update_key_value(outputs=outputs, targets=targets)
+        for key in update_true_value:
+            assert key in intermediate_metrics
+            assert abs(intermediate_metrics[key] - update_true_value[key]) < EPS
+    metrics = metric.compute_key_value()
+    for key in compute_true_value:
+        assert key in metrics
+        assert abs(metrics[key] - compute_true_value[key]) < EPS
+
+
 @pytest.mark.parametrize(
     "outputs_list,targets_list,zero_division,update_true_values,compute_true_value",
     (