Repository: Project-MONAI/MONAI

Issue:
Incorrect implementation of generalized Dice loss
In the line 195 of dice.py the element-wise division cancels out the weights (if `smooth` is negligible), which destroys the most important property of generalized Dice loss. 

https://github.com/Project-MONAI/MONAI/blob/ad06dff7f85711048690b2e85c99d51001612708/monai/losses/dice.py#L195



Discussion:

Resolving patch:
diff --git a/monai/losses/dice.py b/monai/losses/dice.py
--- a/monai/losses/dice.py
+++ b/monai/losses/dice.py
@@ -20,9 +20,9 @@
 class DiceLoss(_Loss):
     """
     Compute average Dice loss between two tensors. It can support both multi-classes and multi-labels tasks.
-    Input logits `pred` (BNHW[D] where N is number of classes) is compared with ground truth `ground` (BNHW[D]).
-    Axis N of `pred` is expected to have logit predictions for each class rather than being image channels,
-    while the same axis of `ground` can be 1 or N(one-hot format). The `smooth` parameter is a value added to the
+    Input logits `input` (BNHW[D] where N is number of classes) is compared with ground truth `target` (BNHW[D]).
+    Axis N of `input` is expected to have logit predictions for each class rather than being image channels,
+    while the same axis of `target` can be 1 or N (one-hot format). The `smooth` parameter is a value added to the
     intersection and union components of the inter-over-union calculation to smooth results and prevent divide by 0,
     this value should be small. The `include_background` class attribute can be set to False for an instance of
     DiceLoss to exclude the first category (channel index 0) which is by convention assumed to be background.
@@ -38,6 +38,7 @@ def __init__(
         do_softmax=False,
         squared_pred=False,
         jaccard=False,
+        reduction="mean",
     ):
         """
         Args:
@@ -47,9 +48,13 @@ def __init__(
             do_softmax (bool): If True, apply a softmax function to the prediction.
             squared_pred (bool): use squared versions of targets and predictions in the denominator or not.
             jaccard (bool): compute Jaccard Index (soft IoU) instead of dice or not.
-
+            reduction (`none|mean|sum`): Specifies the reduction to apply to the output:
+                ``'none'``: no reduction will be applied,
+                ``'mean'``: the sum of the output will be divided by the number of elements in the output,
+                ``'sum'``: the output will be summed.
+                Default: ``'mean'``.
         """
-        super().__init__()
+        super().__init__(reduction=reduction)
         self.include_background = include_background
         self.to_onehot_y = to_onehot_y
         if do_sigmoid and do_softmax:
@@ -59,16 +64,16 @@ def __init__(
         self.squared_pred = squared_pred
         self.jaccard = jaccard
 
-    def forward(self, pred, ground, smooth=1e-5):
+    def forward(self, input, target, smooth=1e-5):
         """
         Args:
-            pred (tensor): the shape should be BNH[WD].
-            ground (tensor): the shape should be BNH[WD].
+            input (tensor): the shape should be BNH[WD].
+            target (tensor): the shape should be BNH[WD].
             smooth (float): a small constant to avoid nan.
         """
         if self.do_sigmoid:
-            pred = torch.sigmoid(pred)
-        n_pred_ch = pred.shape[1]
+            input = torch.sigmoid(input)
+        n_pred_ch = input.shape[1]
         if n_pred_ch == 1:
             if self.do_softmax:
                 warnings.warn("single channel prediction, `do_softmax=True` ignored.")
@@ -78,36 +83,41 @@ def forward(self, pred, ground, smooth=1e-5):
                 warnings.warn("single channel prediction, `include_background=False` ignored.")
         else:
             if self.do_softmax:
-                pred = torch.softmax(pred, 1)
+                input = torch.softmax(input, 1)
             if self.to_onehot_y:
-                ground = one_hot(ground, n_pred_ch)
+                target = one_hot(target, n_pred_ch)
             if not self.include_background:
                 # if skipping background, removing first channel
-                ground = ground[:, 1:]
-                pred = pred[:, 1:]
-                assert ground.shape == pred.shape, "ground truth one-hot has differing shape (%r) from pred (%r)" % (
-                    ground.shape,
-                    pred.shape,
-                )
+                target = target[:, 1:]
+                input = input[:, 1:]
+        assert (
+            target.shape == input.shape
+        ), f"ground truth has differing shape ({target.shape}) from input ({input.shape})"
 
         # reducing only spatial dimensions (not batch nor channels)
-        reduce_axis = list(range(2, len(pred.shape)))
-        intersection = torch.sum(ground * pred, reduce_axis)
+        reduce_axis = list(range(2, len(input.shape)))
+        intersection = torch.sum(target * input, reduce_axis)
 
         if self.squared_pred:
-            ground = torch.pow(ground, 2)
-            pred = torch.pow(pred, 2)
+            target = torch.pow(target, 2)
+            input = torch.pow(input, 2)
 
-        ground_o = torch.sum(ground, reduce_axis)
-        pred_o = torch.sum(pred, reduce_axis)
+        ground_o = torch.sum(target, reduce_axis)
+        pred_o = torch.sum(input, reduce_axis)
 
         denominator = ground_o + pred_o
 
         if self.jaccard:
             denominator -= intersection
 
-        f = (2.0 * intersection + smooth) / (denominator + smooth)
-        return 1.0 - f.mean()  # final reduce_mean across batches and channels
+        f = 1.0 - (2.0 * intersection + smooth) / (denominator + smooth)
+        if self.reduction == "sum":
+            return f.sum()  # sum over the batch and channel dims
+        if self.reduction == "none":
+            return f  # returns [N, n_classes] losses
+        if self.reduction == "mean":
+            return f.mean()  # the batch and channel average
+        raise ValueError(f"reduction={self.reduction} is invalid.")
 
 
 class GeneralizedDiceLoss(_Loss):
@@ -121,7 +131,15 @@ class GeneralizedDiceLoss(_Loss):
         https://github.com/NifTK/NiftyNet/blob/v0.6.0/niftynet/layer/loss_segmentation.py#L279
     """
 
-    def __init__(self, include_background=True, to_onehot_y=False, do_sigmoid=False, do_softmax=False, w_type="square"):
+    def __init__(
+        self,
+        include_background=True,
+        to_onehot_y=False,
+        do_sigmoid=False,
+        do_softmax=False,
+        w_type="square",
+        reduction="mean",
+    ):
         """
         Args:
             include_background (bool): If False channel index 0 (background category) is excluded from the calculation.
@@ -129,9 +147,14 @@ def __init__(self, include_background=True, to_onehot_y=False, do_sigmoid=False,
             do_sigmoid (bool): If True, apply a sigmoid function to the prediction.
             do_softmax (bool): If True, apply a softmax function to the prediction.
             w_type ('square'|'simple'|'uniform'): type of function to transform ground truth volume to a weight factor.
-
+                Default: `'square'`
+            reduction (`none|mean|sum`): Specifies the reduction to apply to the output:
+                ``'none'``: no reduction will be applied,
+                ``'mean'``: the sum of the output will be divided by the batch size in the output,
+                ``'sum'``: the output will be summed over the batch dim.
+                Default: ``'mean'``.
         """
-        super().__init__()
+        super().__init__(reduction=reduction)
         self.include_background = include_background
         self.to_onehot_y = to_onehot_y
         if do_sigmoid and do_softmax:
@@ -143,19 +166,17 @@ def __init__(self, include_background=True, to_onehot_y=False, do_sigmoid=False,
             self.w_func = torch.reciprocal
         elif w_type == "square":
             self.w_func = lambda x: torch.reciprocal(x * x)
-        else:
-            raise ValueError(f"unknown option for `w_type`: {w_type}")
 
-    def forward(self, pred, ground, smooth=1e-5):
+    def forward(self, input, target, smooth=1e-5):
         """
         Args:
-            pred (tensor): the shape should be BNH[WD].
-            ground (tensor): the shape should be BNH[WD].
+            input (tensor): the shape should be BNH[WD].
+            target (tensor): the shape should be BNH[WD].
             smooth (float): a small constant to avoid nan.
         """
         if self.do_sigmoid:
-            pred = torch.sigmoid(pred)
-        n_pred_ch = pred.shape[1]
+            input = torch.sigmoid(input)
+        n_pred_ch = input.shape[1]
         if n_pred_ch == 1:
             if self.do_softmax:
                 warnings.warn("single channel prediction, `do_softmax=True` ignored.")
@@ -165,24 +186,23 @@ def forward(self, pred, ground, smooth=1e-5):
                 warnings.warn("single channel prediction, `include_background=False` ignored.")
         else:
             if self.do_softmax:
-                pred = torch.softmax(pred, 1)
+                input = torch.softmax(input, 1)
             if self.to_onehot_y:
-                ground = one_hot(ground, n_pred_ch)
+                target = one_hot(target, n_pred_ch)
             if not self.include_background:
                 # if skipping background, removing first channel
-                ground = ground[:, 1:]
-                pred = pred[:, 1:]
-                assert ground.shape == pred.shape, "ground truth one-hot has differing shape (%r) from pred (%r)" % (
-                    ground.shape,
-                    pred.shape,
-                )
+                target = target[:, 1:]
+                input = input[:, 1:]
+        assert (
+            target.shape == input.shape
+        ), f"ground truth has differing shape ({target.shape}) from input ({input.shape})"
 
         # reducing only spatial dimensions (not batch nor channels)
-        reduce_axis = list(range(2, len(pred.shape)))
-        intersection = torch.sum(ground * pred, reduce_axis)
+        reduce_axis = list(range(2, len(input.shape)))
+        intersection = torch.sum(target * input, reduce_axis)
 
-        ground_o = torch.sum(ground, reduce_axis)
-        pred_o = torch.sum(pred, reduce_axis)
+        ground_o = torch.sum(target, reduce_axis)
+        pred_o = torch.sum(input, reduce_axis)
 
         denominator = ground_o + pred_o
 
@@ -192,8 +212,14 @@ def forward(self, pred, ground, smooth=1e-5):
             b[infs] = 0.0
             b[infs] = torch.max(b)
 
-        f = (2.0 * intersection * w + smooth) / (denominator * w + smooth)
-        return 1.0 - f.mean()  # final reduce_mean across batches and channels
+        f = 1.0 - (2.0 * (intersection * w).sum(1) + smooth) / ((denominator * w).sum(1) + smooth)
+        if self.reduction == "sum":
+            return f.sum()  # sum over the batch dim
+        if self.reduction == "none":
+            return f  # returns [N] losses
+        if self.reduction == "mean":
+            return f.mean()  # the batch and channel average
+        raise ValueError(f"reduction={self.reduction} is invalid.")
 
 
 dice = Dice = DiceLoss
diff --git a/monai/losses/focal_loss.py b/monai/losses/focal_loss.py
--- a/monai/losses/focal_loss.py
+++ b/monai/losses/focal_loss.py
@@ -23,20 +23,26 @@ class FocalLoss(_WeightedLoss):
     def __init__(self, gamma=2.0, weight=None, reduction="mean"):
         """
         Args:
-            gamma: (float) value of the exponent gamma in the definition
-            of the Focal loss.
-            weight: (tensor) weights to apply to the
-            voxels of each class. If None no weights are applied.
-            This corresponds to the weights \alpha in [1].
-            reduction: (string) reduction operation to apply on the loss batch.
-            It can be 'mean', 'sum' or 'none' as in the standard PyTorch API
-            for loss functions.
-
-        Exemple:
-            pred = torch.tensor([[1, 0], [0, 1], [1, 0]], dtype=torch.float32)
-            grnd = torch.tensor([0, 1 ,0], dtype=torch.int64)
-            fl = FocalLoss()
-            fl(pred, grnd)
+            gamma (float): value of the exponent gamma in the definition of the Focal loss.
+            weight (tensor): weights to apply to the voxels of each class. If None no weights are applied.
+                This corresponds to the weights `\alpha` in [1].
+            reduction (`none|mean|sum`): Specifies the reduction to apply to the output:
+                ``'none'``: no reduction will be applied,
+                ``'mean'``: the sum of the output will be divided by the batch size in the output,
+                ``'sum'``: the output will be summed over the batch dim.
+                Default: ``'mean'``.
+
+        Example:
+            .. code-block:: python
+
+                import torch
+                from monai.losses import FocalLoss
+
+                pred = torch.tensor([[1, 0], [0, 1], [1, 0]], dtype=torch.float32)
+                grnd = torch.tensor([[0], [1], [0]], dtype=torch.int64)
+                fl = FocalLoss()
+                fl(pred, grnd)
+
         """
         super(FocalLoss, self).__init__(weight=weight, reduction=reduction)
         self.gamma = gamma
@@ -44,16 +50,23 @@ def __init__(self, gamma=2.0, weight=None, reduction="mean"):
     def forward(self, input, target):
         """
         Args:
-            input: (tensor): the shape should be BNH[WD].
-            target: (tensor): the shape should be BNH[WD].
+            input: (tensor): the shape should be BCH[WD].
+                where C is the number of classes.
+            target: (tensor): the shape should be B1H[WD].
+                The target that this loss expects should be a class index in the range
+                [0, C-1] where C is the number of classes.
         """
         i = input
         t = target
 
-        if t.dim() < i.dim():
-            # Add a class dimension to the ground-truth segmentation.
-            t = t.unsqueeze(1)  # N,H,W => N,1,H,W
+        if i.ndim != t.ndim:
+            raise ValueError(f"input and target must have the same number of dimensions, got {i.ndim} and {t.ndim}")
 
+        if target.shape[1] != 1:
+            raise ValueError(
+                f"target must have one channel, and should be a class index in the range [0, C-1] "
+                + f"where C is the number of classes inferred from 'input': C={i.shape[1]}."
+            )
         # Change the shape of input and target to
         # num_batch x num_class x num_voxels.
         if input.dim() > 2:
@@ -66,21 +79,20 @@ def forward(self, input, target):
         # Compute the log proba (more stable numerically than softmax).
         logpt = F.log_softmax(i, dim=1)  # N,C,H*W
         # Keep only log proba values of the ground truth class for each voxel.
-        logpt = logpt.gather(1, t)  # N,C,H*W => N,1,H*W
+        logpt = logpt.gather(1, t.long())  # N,C,H*W => N,1,H*W
         logpt = torch.squeeze(logpt, dim=1)  # N,1,H*W => N,H*W
 
         # Get the proba
         pt = torch.exp(logpt)  # N,H*W
 
         if self.weight is not None:
-            if self.weight.type() != i.data.type():
-                self.weight = self.weight.type_as(i.data)
+            self.weight = self.weight.to(i)
             # Convert the weight to a map in which each voxel
             # has the weight associated with the ground-truth label
             # associated with this voxel in target.
             at = self.weight[None, :, None]  # C => 1,C,1
             at = at.expand((t.size(0), -1, t.size(2)))  # 1,C,1 => N,C,H*W
-            at = at.gather(1, t.data)  # selection of the weights  => N,1,H*W
+            at = at.gather(1, t.long())  # selection of the weights  => N,1,H*W
             at = torch.squeeze(at, dim=1)  # N,1,H*W => N,H*W
             # Multiply the log proba by their weights.
             logpt = logpt * at
@@ -91,8 +103,8 @@ def forward(self, input, target):
 
         if self.reduction == "sum":
             return loss.sum()
-        elif self.reduction == "none":
+        if self.reduction == "none":
             return loss
-        # Default is mean reduction.
-        else:
+        if self.reduction == "mean":
             return loss.mean()
+        raise ValueError(f"reduction={self.reduction} is invalid.")
diff --git a/monai/losses/tversky.py b/monai/losses/tversky.py
--- a/monai/losses/tversky.py
+++ b/monai/losses/tversky.py
@@ -20,7 +20,7 @@
 class TverskyLoss(_Loss):
 
     """
-    Compute the Tversky loss defined in:        
+    Compute the Tversky loss defined in:
 
         Sadegh et al. (2017) Tversky loss function for image segmentation
         using 3D fully convolutional deep networks. (https://arxiv.org/abs/1706.05721)
@@ -31,7 +31,14 @@ class TverskyLoss(_Loss):
     """
 
     def __init__(
-        self, include_background=True, to_onehot_y=False, do_sigmoid=False, do_softmax=False, alpha=0.5, beta=0.5
+        self,
+        include_background=True,
+        to_onehot_y=False,
+        do_sigmoid=False,
+        do_softmax=False,
+        alpha=0.5,
+        beta=0.5,
+        reduction="mean",
     ):
 
         """
@@ -42,9 +49,15 @@ def __init__(
             do_softmax (bool): If True, apply a softmax function to the prediction.
             alpha (float): weight of false positives
             beta  (float): weight of false negatives
+            reduction (`none|mean|sum`): Specifies the reduction to apply to the output:
+                ``'none'``: no reduction will be applied,
+                ``'mean'``: the sum of the output will be divided by the number of elements in the output,
+                ``'sum'``: the output will be summed.
+                Default: ``'mean'``.
+
         """
 
-        super().__init__()
+        super().__init__(reduction=reduction)
         self.include_background = include_background
         self.to_onehot_y = to_onehot_y
 
@@ -55,16 +68,16 @@ def __init__(
         self.alpha = alpha
         self.beta = beta
 
-    def forward(self, pred, ground, smooth=1e-5):
+    def forward(self, input, target, smooth=1e-5):
         """
         Args:
-            pred (tensor): the shape should be BNH[WD].
-            ground (tensor): the shape should be BNH[WD].
+            input (tensor): the shape should be BNH[WD].
+            target (tensor): the shape should be BNH[WD].
             smooth (float): a small constant to avoid nan.
         """
         if self.do_sigmoid:
-            pred = torch.sigmoid(pred)
-        n_pred_ch = pred.shape[1]
+            input = torch.sigmoid(input)
+        n_pred_ch = input.shape[1]
         if n_pred_ch == 1:
             if self.do_softmax:
                 warnings.warn("single channel prediction, `do_softmax=True` ignored.")
@@ -74,25 +87,24 @@ def forward(self, pred, ground, smooth=1e-5):
                 warnings.warn("single channel prediction, `include_background=False` ignored.")
         else:
             if self.do_softmax:
-                pred = torch.softmax(pred, 1)
+                input = torch.softmax(input, 1)
             if self.to_onehot_y:
-                ground = one_hot(ground, n_pred_ch)
+                target = one_hot(target, n_pred_ch)
             if not self.include_background:
                 # if skipping background, removing first channel
-                ground = ground[:, 1:]
-                pred = pred[:, 1:]
-                assert ground.shape == pred.shape, "ground truth one-hot has differing shape (%r) from pred (%r)" % (
-                    ground.shape,
-                    pred.shape,
-                )
-
-        p0 = pred
+                target = target[:, 1:]
+                input = input[:, 1:]
+        assert (
+            target.shape == input.shape
+        ), f"ground truth has differing shape ({target.shape}) from input ({input.shape})"
+
+        p0 = input
         p1 = 1 - p0
-        g0 = ground
+        g0 = target
         g1 = 1 - g0
 
         # reducing only spatial dimensions (not batch nor channels)
-        reduce_axis = list(range(2, len(pred.shape)))
+        reduce_axis = list(range(2, len(input.shape)))
 
         tp = torch.sum(p0 * g0, reduce_axis)
         fp = self.alpha * torch.sum(p0 * g1, reduce_axis)
@@ -101,6 +113,12 @@ def forward(self, pred, ground, smooth=1e-5):
         numerator = tp + smooth
         denominator = tp + fp + fn + smooth
 
-        score = numerator / denominator
+        score = 1.0 - numerator / denominator
 
-        return 1.0 - score.mean()
+        if self.reduction == "sum":
+            return score.sum()  # sum over the batch and channel dims
+        if self.reduction == "none":
+            return score  # returns [N, n_classes] losses
+        if self.reduction == "mean":
+            return score.mean()
+        raise ValueError(f"reduction={self.reduction} is invalid.")


Test patch:
diff --git a/tests/__init__.py b/tests/__init__.py
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -8,3 +8,29 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import sys
+import unittest
+import warnings
+
+
+def _enter_pr_4800(self):
+    """
+    code from https://github.com/python/cpython/pull/4800
+    """
+    # The __warningregistry__'s need to be in a pristine state for tests
+    # to work properly.
+    for v in list(sys.modules.values()):
+        if getattr(v, "__warningregistry__", None):
+            v.__warningregistry__ = {}
+    self.warnings_manager = warnings.catch_warnings(record=True)
+    self.warnings = self.warnings_manager.__enter__()
+    warnings.simplefilter("always", self.expected)
+    return self
+
+
+# workaround for https://bugs.python.org/issue29620
+try:
+    unittest.case._AssertWarnsContext.__enter__ = _enter_pr_4800
+except AttributeError:
+    pass
diff --git a/tests/test_dice_loss.py b/tests/test_dice_loss.py
--- a/tests/test_dice_loss.py
+++ b/tests/test_dice_loss.py
@@ -11,99 +11,139 @@
 
 import unittest
 
+import numpy as np
 import torch
 from parameterized import parameterized
 
 from monai.losses import DiceLoss
 
-TEST_CASE_1 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
-        "smooth": 1e-6,
-    },
-    0.307576,
-]
-
-TEST_CASE_2 = [  # shape: (2, 1, 2, 2), (2, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]], [[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 1.0], [1.0, 1.0]]], [[[1.0, 0.0], [1.0, 0.0]]]]),
-        "smooth": 1e-4,
-    },
-    0.416657,
-]
-
-TEST_CASE_3 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": False, "to_onehot_y": True},
-    {
-        "pred": torch.tensor([[[1.0, 1.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
-        "ground": torch.tensor([[[0.0, 0.0, 1.0]], [[0.0, 1.0, 0.0]]]),
-        "smooth": 0.0,
-    },
-    0.0,
-]
-
-TEST_CASE_4 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": True, "to_onehot_y": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
-        "ground": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
-        "smooth": 1e-4,
-    },
-    0.435050,
-]
-
-TEST_CASE_5 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": True, "to_onehot_y": True, "do_softmax": True},
-    {
-        "pred": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
-        "ground": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
-        "smooth": 1e-4,
-    },
-    0.383713,
-]
-
-TEST_CASE_6 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
-        "smooth": 1e-6,
-    },
-    0.307576,
-]
-
-TEST_CASE_7 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True, "squared_pred": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
-        "smooth": 1e-5,
-    },
-    0.178337,
-]
-
-TEST_CASE_8 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True, "jaccard": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
-        "smooth": 1e-5,
-    },
-    -0.059094,
+TEST_CASES = [
+    [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+            "smooth": 1e-6,
+        },
+        0.307576,
+    ],
+    [  # shape: (2, 1, 2, 2), (2, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]], [[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 1.0], [1.0, 1.0]]], [[[1.0, 0.0], [1.0, 0.0]]]]),
+            "smooth": 1e-4,
+        },
+        0.416657,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": False, "to_onehot_y": True},
+        {
+            "input": torch.tensor([[[1.0, 1.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
+            "target": torch.tensor([[[0.0, 0.0, 1.0]], [[0.0, 1.0, 0.0]]]),
+            "smooth": 0.0,
+        },
+        0.0,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        0.435050,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_sigmoid": True, "reduction": "none"},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        [[0.296529, 0.415136], [0.599976, 0.428559]],
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_softmax": True},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        0.383713,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_softmax": True, "reduction": "sum"},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        1.534853,
+    ],
+    [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+            "smooth": 1e-6,
+        },
+        0.307576,
+    ],
+    [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True, "squared_pred": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+            "smooth": 1e-5,
+        },
+        0.178337,
+    ],
+    [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True, "jaccard": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+            "smooth": 1e-5,
+        },
+        -0.059094,
+    ],
 ]
 
 
 class TestDiceLoss(unittest.TestCase):
-    @parameterized.expand(
-        [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]
-    )
+    @parameterized.expand(TEST_CASES)
     def test_shape(self, input_param, input_data, expected_val):
         result = DiceLoss(**input_param).forward(**input_data)
-        self.assertAlmostEqual(result.item(), expected_val, places=5)
+        np.testing.assert_allclose(result.detach().cpu().numpy(), expected_val, rtol=1e-5)
+
+    def test_ill_shape(self):
+        loss = DiceLoss()
+        with self.assertRaisesRegex(AssertionError, ""):
+            loss.forward(torch.ones((1, 2, 3)), torch.ones((4, 5, 6)))
+
+    def test_ill_opts(self):
+        with self.assertRaisesRegex(ValueError, ""):
+            DiceLoss(do_sigmoid=True, do_softmax=True)
+        chn_input = torch.ones((1, 1, 3))
+        chn_target = torch.ones((1, 1, 3))
+        with self.assertRaisesRegex(ValueError, ""):
+            DiceLoss(reduction="unknown")(chn_input, chn_target)
+        with self.assertRaisesRegex(ValueError, ""):
+            DiceLoss(reduction=None)(chn_input, chn_target)
+
+    def test_input_warnings(self):
+        chn_input = torch.ones((1, 1, 3))
+        chn_target = torch.ones((1, 1, 3))
+        with self.assertWarns(Warning):
+            loss = DiceLoss(include_background=False)
+            loss.forward(chn_input, chn_target)
+        with self.assertWarns(Warning):
+            loss = DiceLoss(do_softmax=True)
+            loss.forward(chn_input, chn_target)
+        with self.assertWarns(Warning):
+            loss = DiceLoss(to_onehot_y=True)
+            loss.forward(chn_input, chn_target)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_focal_loss.py b/tests/test_focal_loss.py
--- a/tests/test_focal_loss.py
+++ b/tests/test_focal_loss.py
@@ -9,11 +9,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import torch.nn.functional as F
-import torch.nn as nn
-import torch.optim as optim
-import torch
 import unittest
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
 from monai.losses import FocalLoss
 
 
@@ -29,13 +30,12 @@ def test_consistency_with_cross_entropy_2d(self):
             # Create a random tensor of shape (batch_size, class_num, 8, 4)
             x = torch.rand(batch_size, class_num, 8, 4, requires_grad=True)
             # Create a random batch of classes
-            l = torch.randint(low=0, high=class_num, size=(batch_size, 8, 4))
-            l = l.long()
+            l = torch.randint(low=0, high=class_num, size=(batch_size, 1, 8, 4))
             if torch.cuda.is_available():
                 x = x.cuda()
                 l = l.cuda()
             output0 = focal_loss.forward(x, l)
-            output1 = ce.forward(x, l)
+            output1 = ce.forward(x, l[:, 0])
             a = float(output0.cpu().detach())
             b = float(output1.cpu().detach())
             if abs(a - b) > max_error:
@@ -53,13 +53,13 @@ def test_consistency_with_cross_entropy_classification(self):
             # Create a random scores tensor of shape (batch_size, class_num)
             x = torch.rand(batch_size, class_num, requires_grad=True)
             # Create a random batch of classes
-            l = torch.randint(low=0, high=class_num, size=(batch_size,))
+            l = torch.randint(low=0, high=class_num, size=(batch_size, 1))
             l = l.long()
             if torch.cuda.is_available():
                 x = x.cuda()
                 l = l.cuda()
             output0 = focal_loss.forward(x, l)
-            output1 = ce.forward(x, l)
+            output1 = ce.forward(x, l[:, 0])
             a = float(output0.cpu().detach())
             b = float(output1.cpu().detach())
             if abs(a - b) > max_error:
@@ -77,10 +77,6 @@ def test_bin_seg_2d(self):
         loss = FocalLoss()
 
         # focal loss for pred_very_good should be close to 0
-        focal_loss_good = float(loss.forward(pred_very_good, target).cpu())
-        self.assertAlmostEqual(focal_loss_good, 0.0, places=3)
-
-        # Same test, but for target with a class dimension
         target = target.unsqueeze(1)  # shape (1, 1, H, W)
         focal_loss_good = float(loss.forward(pred_very_good, target).cpu())
         self.assertAlmostEqual(focal_loss_good, 0.0, places=3)
@@ -97,6 +93,7 @@ def test_empty_class_2d(self):
         loss = FocalLoss()
 
         # focal loss for pred_very_good should be close to 0
+        target = target.unsqueeze(1)  # shape (1, 1, H, W)
         focal_loss_good = float(loss.forward(pred_very_good, target).cpu())
         self.assertAlmostEqual(focal_loss_good, 0.0, places=3)
 
@@ -112,6 +109,7 @@ def test_multi_class_seg_2d(self):
         loss = FocalLoss()
 
         # focal loss for pred_very_good should be close to 0
+        target = target.unsqueeze(1)  # shape (1, 1, H, W)
         focal_loss_good = float(loss.forward(pred_very_good, target).cpu())
         self.assertAlmostEqual(focal_loss_good, 0.0, places=3)
 
@@ -135,83 +133,26 @@ def test_bin_seg_3d(self):
         loss = FocalLoss()
 
         # focal loss for pred_very_good should be close to 0
+        target = target.unsqueeze(1)  # shape (1, 1, H, W)
         focal_loss_good = float(loss.forward(pred_very_good, target).cpu())
         self.assertAlmostEqual(focal_loss_good, 0.0, places=3)
 
-    def test_convergence(self):
-        """
-        The goal of this test is to assess if the gradient of the loss function
-        is correct by testing if we can train a one layer neural network
-        to segment one image.
-        We verify that the loss is decreasing in almost all SGD steps.
-        """
-        learning_rate = 0.001
-        max_iter = 20
-
-        # define a simple 3d example
-        target_seg = torch.tensor(
-            [
-                # raw 0
-                [[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]],
-                # raw 1
-                [[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]],
-                # raw 2
-                [[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]],
-            ]
-        )
-        target_seg = torch.unsqueeze(target_seg, dim=0)
-        image = 12 * target_seg + 27
-        image = image.float()
-        num_classes = 2
-        num_voxels = 3 * 4 * 4
-
-        # define a one layer model
-        class OnelayerNet(nn.Module):
-            def __init__(self):
-                super(OnelayerNet, self).__init__()
-                self.layer = nn.Linear(num_voxels, num_voxels * num_classes)
-
-            def forward(self, x):
-                x = x.view(-1, num_voxels)
-                x = self.layer(x)
-                x = x.view(-1, num_classes, 3, 4, 4)
-                return x
-
-        # initialise the network
-        net = OnelayerNet()
-
-        # initialize the loss
-        loss = FocalLoss()
-
-        # initialize an SGD
-        optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9)
-
-        loss_history = []
-        # train the network
-        for _ in range(max_iter):
-            # set the gradient to zero
-            optimizer.zero_grad()
-
-            # forward pass
-            output = net(image)
-            loss_val = loss(output, target_seg)
-
-            # backward pass
-            loss_val.backward()
-            optimizer.step()
-
-            # stats
-            loss_history.append(loss_val.item())
-
-        # count the number of SGD steps in which the loss decreases
-        num_decreasing_steps = 0
-        for i in range(len(loss_history) - 1):
-            if loss_history[i] > loss_history[i + 1]:
-                num_decreasing_steps += 1
-        decreasing_steps_ratio = float(num_decreasing_steps) / (len(loss_history) - 1)
-
-        # verify that the loss is decreasing for sufficiently many SGD steps
-        self.assertTrue(decreasing_steps_ratio > 0.9)
+    def test_ill_opts(self):
+        chn_input = torch.ones((1, 2, 3))
+        chn_target = torch.ones((1, 1, 3))
+        with self.assertRaisesRegex(ValueError, ""):
+            FocalLoss(reduction="unknown")(chn_input, chn_target)
+        with self.assertRaisesRegex(ValueError, ""):
+            FocalLoss(reduction=None)(chn_input, chn_target)
+
+    def test_ill_shape(self):
+        chn_input = torch.ones((1, 2, 3))
+        chn_target = torch.ones((1, 3))
+        with self.assertRaisesRegex(ValueError, ""):
+            FocalLoss(reduction="mean")(chn_input, chn_target)
+        chn_target = torch.ones((1, 2, 3))
+        with self.assertRaisesRegex(ValueError, ""):
+            FocalLoss(reduction="mean")(chn_input, chn_target)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_generalized_dice_loss.py b/tests/test_generalized_dice_loss.py
--- a/tests/test_generalized_dice_loss.py
+++ b/tests/test_generalized_dice_loss.py
@@ -11,87 +11,139 @@
 
 import unittest
 
+import numpy as np
 import torch
 from parameterized import parameterized
 
 from monai.losses import GeneralizedDiceLoss
 
-TEST_CASE_0 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
-        "smooth": 1e-6,
-    },
-    0.307576,
-]
-
-TEST_CASE_1 = [  # shape: (2, 1, 2, 2), (2, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]], [[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 1.0], [1.0, 1.0]]], [[[1.0, 0.0], [1.0, 0.0]]]]),
-        "smooth": 1e-4,
-    },
-    0.416597,
-]
-
-TEST_CASE_2 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": False, "to_onehot_y": True},
-    {
-        "pred": torch.tensor([[[1.0, 1.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
-        "ground": torch.tensor([[[0.0, 0.0, 1.0]], [[0.0, 1.0, 0.0]]]),
-        "smooth": 0.0,
-    },
-    0.0,
-]
-
-TEST_CASE_3 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": True, "to_onehot_y": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
-        "ground": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
-        "smooth": 1e-4,
-    },
-    0.435034,
-]
-
-TEST_CASE_4 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": True, "to_onehot_y": True, "do_softmax": True},
-    {
-        "pred": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
-        "ground": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
-        "smooth": 1e-4,
-    },
-    0.383699,
-]
-
-TEST_CASE_5 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": False, "to_onehot_y": True},
-    {
-        "pred": torch.tensor([[[1.0, 1.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
-        "ground": torch.tensor([[[0.0, 0.0, 0.0]], [[0.0, 0.0, 0.0]]]),
-        "smooth": 1e-8,
-    },
-    0.0,
-]
-
-TEST_CASE_6 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
-        "smooth": 1e-6,
-    },
-    0.307576,
+TEST_CASES = [
+    [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+            "smooth": 1e-6,
+        },
+        0.307576,
+    ],
+    [  # shape: (2, 1, 2, 2), (2, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]], [[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 1.0], [1.0, 1.0]]], [[[1.0, 0.0], [1.0, 0.0]]]]),
+            "smooth": 1e-4,
+        },
+        0.416597,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": False, "to_onehot_y": True},
+        {
+            "input": torch.tensor([[[1.0, 1.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
+            "target": torch.tensor([[[0.0, 0.0, 1.0]], [[0.0, 1.0, 0.0]]]),
+            "smooth": 0.0,
+        },
+        0.0,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        0.469964,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_softmax": True},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        0.414507,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_softmax": True, "reduction": "sum"},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        0.829015,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_softmax": True, "reduction": "none"},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        [0.273476, 0.555539],
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": False, "to_onehot_y": True},
+        {
+            "input": torch.tensor([[[1.0, 1.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
+            "target": torch.tensor([[[0.0, 0.0, 0.0]], [[0.0, 0.0, 0.0]]]),
+            "smooth": 1e-8,
+        },
+        0.0,
+    ],
+    [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+            "smooth": 1e-6,
+        },
+        0.307576,
+    ],
+    [  # shape: (1, 2, 4), (1, 1, 4)
+        {"include_background": True, "to_onehot_y": True, "do_softmax": True, "w_type": "simple"},
+        {
+            "input": torch.tensor([[[0.0, 10.0, 10.0, 10.0], [10.0, 0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1, 1, 0, 0]]]),
+            "smooth": 0.0,
+        },
+        0.250023,
+    ],
 ]
 
 
 class TestGeneralizedDiceLoss(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6])
+    @parameterized.expand(TEST_CASES)
     def test_shape(self, input_param, input_data, expected_val):
         result = GeneralizedDiceLoss(**input_param).forward(**input_data)
-        self.assertAlmostEqual(result.item(), expected_val, places=5)
+        np.testing.assert_allclose(result.detach().cpu().numpy(), expected_val, rtol=1e-5)
+
+    def test_ill_shape(self):
+        loss = GeneralizedDiceLoss()
+        with self.assertRaisesRegex(AssertionError, ""):
+            loss.forward(torch.ones((1, 2, 3)), torch.ones((4, 5, 6)))
+
+    def test_ill_opts(self):
+        with self.assertRaisesRegex(ValueError, ""):
+            GeneralizedDiceLoss(do_sigmoid=True, do_softmax=True)
+        chn_input = torch.ones((1, 1, 3))
+        chn_target = torch.ones((1, 1, 3))
+        with self.assertRaisesRegex(ValueError, ""):
+            GeneralizedDiceLoss(reduction="unknown")(chn_input, chn_target)
+        with self.assertRaisesRegex(ValueError, ""):
+            GeneralizedDiceLoss(reduction=None)(chn_input, chn_target)
+
+    def test_input_warnings(self):
+        chn_input = torch.ones((1, 1, 3))
+        chn_target = torch.ones((1, 1, 3))
+        with self.assertWarns(Warning):
+            loss = GeneralizedDiceLoss(include_background=False)
+            loss.forward(chn_input, chn_target)
+        with self.assertWarns(Warning):
+            loss = GeneralizedDiceLoss(do_softmax=True)
+            loss.forward(chn_input, chn_target)
+        with self.assertWarns(Warning):
+            loss = GeneralizedDiceLoss(to_onehot_y=True)
+            loss.forward(chn_input, chn_target)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_seg_loss_integration.py b/tests/test_seg_loss_integration.py
new file mode 100644
--- /dev/null
+++ b/tests/test_seg_loss_integration.py
@@ -0,0 +1,140 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from parameterized import parameterized
+
+from monai.losses import DiceLoss, FocalLoss, GeneralizedDiceLoss, TverskyLoss
+
+TEST_CASES = [
+    [DiceLoss, {"to_onehot_y": True, "squared_pred": True}, {"smooth": 1e-4}],
+    [DiceLoss, {"to_onehot_y": True, "do_sigmoid": True}, {}],
+    [DiceLoss, {"to_onehot_y": True, "do_softmax": True}, {}],
+    [FocalLoss, {"gamma": 1.5, "weight": torch.tensor([1, 2])}, {}],
+    [FocalLoss, {"gamma": 1.5}, {}],
+    [GeneralizedDiceLoss, {"to_onehot_y": True, "do_softmax": True}, {}],
+    [GeneralizedDiceLoss, {"to_onehot_y": True, "do_sigmoid": True}, {}],
+    [GeneralizedDiceLoss, {"to_onehot_y": True, "do_sigmoid": True, "w_type": "simple"}, {}],
+    [GeneralizedDiceLoss, {"to_onehot_y": True, "do_sigmoid": True, "w_type": "uniform"}, {}],
+    [TverskyLoss, {"to_onehot_y": True, "do_softmax": True, "alpha": 0.8, "beta": 0.2}, {}],
+    [TverskyLoss, {"to_onehot_y": True, "do_softmax": True, "alpha": 1.0, "beta": 0.0}, {}],
+]
+
+
+class TestSegLossIntegration(unittest.TestCase):
+    def setUp(self):
+        torch.backends.cudnn.deterministic = True
+        torch.backends.cudnn.benchmark = False
+        torch.manual_seed(0)
+        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu:0")
+
+    def tearDown(self):
+        torch.backends.cudnn.deterministic = False
+        torch.backends.cudnn.benchmark = True
+
+    @parameterized.expand(TEST_CASES)
+    def test_convergence(self, loss_type, loss_args, forward_args):
+        """
+        The goal of this test is to assess if the gradient of the loss function
+        is correct by testing if we can train a one layer neural network
+        to segment one image.
+        We verify that the loss is decreasing in almost all SGD steps.
+        """
+        learning_rate = 0.001
+        max_iter = 40
+
+        # define a simple 3d example
+        target_seg = torch.tensor(
+            [
+                [
+                    # raw 0
+                    [[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]],
+                    # raw 1
+                    [[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]],
+                    # raw 2
+                    [[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]],
+                ]
+            ],
+            device=self.device,
+        )
+        target_seg = torch.unsqueeze(target_seg, dim=0)
+        image = 12 * target_seg + 27
+        image = image.float().to(self.device)
+        num_classes = 2
+        num_voxels = 3 * 4 * 4
+
+        # define a one layer model
+        class OnelayerNet(nn.Module):
+            def __init__(self):
+                super(OnelayerNet, self).__init__()
+                self.layer_1 = nn.Linear(num_voxels, 200)
+                self.acti = nn.ReLU()
+                self.layer_2 = nn.Linear(200, num_voxels * num_classes)
+
+            def forward(self, x):
+                x = x.view(-1, num_voxels)
+                x = self.layer_1(x)
+                x = self.acti(x)
+                x = self.layer_2(x)
+                x = x.view(-1, num_classes, 3, 4, 4)
+                return x
+
+        # initialise the network
+        net = OnelayerNet().to(self.device)
+
+        # initialize the loss
+        loss = loss_type(**loss_args)
+
+        # initialize a SGD optimizer
+        optimizer = optim.Adam(net.parameters(), lr=learning_rate)
+
+        loss_history = []
+        init_output = None
+
+        # train the network
+        for iter_i in range(max_iter):
+            # set the gradient to zero
+            optimizer.zero_grad()
+
+            # forward pass
+            output = net(image)
+            if init_output is None:
+                init_output = torch.argmax(output, 1).detach().cpu().numpy()
+
+            loss_val = loss(output, target_seg, **forward_args)
+
+            if iter_i % 10 == 0:
+                pred = torch.argmax(output, 1).detach().cpu().numpy()
+                gt = target_seg.detach().cpu().numpy()[:, 0]
+                print(f"{loss_type.__name__} iter: {iter_i}, acc: {np.sum(pred == gt) / np.prod(pred.shape)}")
+
+            # backward pass
+            loss_val.backward()
+            optimizer.step()
+
+            # stats
+            loss_history.append(loss_val.item())
+
+        pred = torch.argmax(output, 1).detach().cpu().numpy()
+        target = target_seg.detach().cpu().numpy()[:, 0]
+        # initial predictions are bad
+        self.assertTrue(not np.allclose(init_output, target))
+        # final predictions are good
+        np.testing.assert_allclose(pred, target)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_tversky_loss.py b/tests/test_tversky_loss.py
--- a/tests/test_tversky_loss.py
+++ b/tests/test_tversky_loss.py
@@ -11,87 +11,126 @@
 
 import unittest
 
+import numpy as np
 import torch
 from parameterized import parameterized
 
 from monai.losses import TverskyLoss
 
-TEST_CASE_1 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
-        "smooth": 1e-6,
-    },
-    0.307576,
-]
-
-TEST_CASE_2 = [  # shape: (2, 1, 2, 2), (2, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]], [[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 1.0], [1.0, 1.0]]], [[[1.0, 0.0], [1.0, 0.0]]]]),
-        "smooth": 1e-4,
-    },
-    0.416657,
-]
-
-TEST_CASE_3 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": False, "to_onehot_y": True},
-    {
-        "pred": torch.tensor([[[1.0, 1.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
-        "ground": torch.tensor([[[0.0, 0.0, 1.0]], [[0.0, 1.0, 0.0]]]),
-        "smooth": 0.0,
-    },
-    0.0,
-]
-
-TEST_CASE_4 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": True, "to_onehot_y": True, "do_sigmoid": True},
-    {
-        "pred": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
-        "ground": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
-        "smooth": 1e-4,
-    },
-    0.435050,
-]
-
-TEST_CASE_5 = [  # shape: (2, 2, 3), (2, 1, 3)
-    {"include_background": True, "to_onehot_y": True, "do_softmax": True},
-    {
-        "pred": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
-        "ground": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
-        "smooth": 1e-4,
-    },
-    0.383713,
-]
-
-TEST_CASE_6 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True, "alpha": 0.3, "beta": 0.7},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
-        "smooth": 1e-6,
-    },
-    0.3589,
-]
-
-TEST_CASE_7 = [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
-    {"include_background": True, "do_sigmoid": True, "alpha": 0.7, "beta": 0.3},
-    {
-        "pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
-        "ground": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
-        "smooth": 1e-6,
-    },
-    0.2474,
+TEST_CASES = [
+    [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+            "smooth": 1e-6,
+        },
+        0.307576,
+    ],
+    [  # shape: (2, 1, 2, 2), (2, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]], [[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 1.0], [1.0, 1.0]]], [[[1.0, 0.0], [1.0, 0.0]]]]),
+            "smooth": 1e-4,
+        },
+        0.416657,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": False, "to_onehot_y": True},
+        {
+            "input": torch.tensor([[[1.0, 1.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
+            "target": torch.tensor([[[0.0, 0.0, 1.0]], [[0.0, 1.0, 0.0]]]),
+            "smooth": 0.0,
+        },
+        0.0,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_sigmoid": True},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        0.435050,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_sigmoid": True, "reduction": "sum"},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        1.74013,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_softmax": True},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        0.383713,
+    ],
+    [  # shape: (2, 2, 3), (2, 1, 3)
+        {"include_background": True, "to_onehot_y": True, "do_softmax": True, "reduction": "none"},
+        {
+            "input": torch.tensor([[[-1.0, 0.0, 1.0], [1.0, 0.0, -1.0]], [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]),
+            "target": torch.tensor([[[1.0, 0.0, 0.0]], [[1.0, 1.0, 0.0]]]),
+            "smooth": 1e-4,
+        },
+        [[0.210961, 0.295339], [0.599952, 0.428547]],
+    ],
+    [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True, "alpha": 0.3, "beta": 0.7},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+            "smooth": 1e-6,
+        },
+        0.3589,
+    ],
+    [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
+        {"include_background": True, "do_sigmoid": True, "alpha": 0.7, "beta": 0.3},
+        {
+            "input": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]),
+            "target": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+            "smooth": 1e-6,
+        },
+        0.247366,
+    ],
 ]
 
 
 class TestTverskyLoss(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7])
+    @parameterized.expand(TEST_CASES)
     def test_shape(self, input_param, input_data, expected_val):
         result = TverskyLoss(**input_param).forward(**input_data)
-        self.assertAlmostEqual(result.item(), expected_val, places=4)
+        np.testing.assert_allclose(result.detach().cpu().numpy(), expected_val, rtol=1e-4)
+
+    def test_ill_shape(self):
+        loss = TverskyLoss()
+        with self.assertRaisesRegex(AssertionError, ""):
+            loss.forward(torch.ones((2, 2, 3)), torch.ones((4, 5, 6)))
+        chn_input = torch.ones((1, 1, 3))
+        chn_target = torch.ones((1, 1, 3))
+        with self.assertRaisesRegex(ValueError, ""):
+            TverskyLoss(reduction="unknown")(chn_input, chn_target)
+        with self.assertRaisesRegex(ValueError, ""):
+            TverskyLoss(reduction=None)(chn_input, chn_target)
+
+    def test_input_warnings(self):
+        chn_input = torch.ones((1, 1, 3))
+        chn_target = torch.ones((1, 1, 3))
+        with self.assertWarns(Warning):
+            loss = TverskyLoss(include_background=False)
+            loss.forward(chn_input, chn_target)
+        with self.assertWarns(Warning):
+            loss = TverskyLoss(do_softmax=True)
+            loss.forward(chn_input, chn_target)
+        with self.assertWarns(Warning):
+            loss = TverskyLoss(to_onehot_y=True)
+            loss.forward(chn_input, chn_target)
 
 
 if __name__ == "__main__":
diff --git a/tests/utils.py b/tests/utils.py
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -10,10 +10,10 @@
 # limitations under the License.
 
 import os
-import unittest
 import tempfile
-import nibabel as nib
+import unittest
 
+import nibabel as nib
 import numpy as np
 import torch