Repository: deepchecks/deepchecks

Issue:
[FEAT] [CV] Add testing to class performance
Very low coverage - 46%


Discussion:
noticed that has no tests for detection either - only tests with mnist

Resolving patch:
diff --git a/deepchecks/vision/checks/performance/class_performance.py b/deepchecks/vision/checks/performance/class_performance.py
--- a/deepchecks/vision/checks/performance/class_performance.py
+++ b/deepchecks/vision/checks/performance/class_performance.py
@@ -261,7 +261,7 @@ def condition(check_result: pd.DataFrame) -> ConditionResult:
 
             datasets_details = []
             for dataset in ['Test', 'Train']:
-                data = check_result.loc[check_result['Dataset'] == dataset].loc[check_result['Metric'] == score]
+                data = check_result.loc[(check_result['Dataset'] == dataset) & (check_result['Metric'] == score)]
 
                 min_value_index = data['Value'].idxmin()
                 min_row = data.loc[min_value_index]


Test patch:
diff --git a/tests/vision/checks/performance/class_performance_test.py b/tests/vision/checks/performance/class_performance_test.py
--- a/tests/vision/checks/performance/class_performance_test.py
+++ b/tests/vision/checks/performance/class_performance_test.py
@@ -12,7 +12,8 @@
 from deepchecks.vision.checks.performance.class_performance import ClassPerformance
 
 from ignite.metrics import Precision, Recall
-from hamcrest import assert_that, calling, close_to, equal_to, is_in, raises
+from hamcrest import assert_that, calling, close_to, equal_to, is_in, raises, is_
+
 
 def test_mnist_average_error_error(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist, device):
     # Arrange
@@ -26,6 +27,7 @@ def test_mnist_average_error_error(mnist_dataset_train, mnist_dataset_test, mock
                    r'The metric p returned a <class \'float\'> instead of an array/tensor')
     )
 
+
 def test_mnist_largest(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist, device):
     # Arrange
     check = ClassPerformance(n_to_show=2, show_only='largest')
@@ -136,3 +138,96 @@ def test_class_list(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
     assert_that(result.value['Class'].iloc[1], equal_to(1))
     assert_that(result.value['Class'].iloc[2], equal_to(1))
     assert_that(result.value['Class'].iloc[3], equal_to(1))
+
+
+def test_condition_test_performance_not_less_than_pass(mnist_dataset_train,
+                                                       mnist_dataset_test,
+                                                       mock_trained_mnist,
+                                                       device):
+    # Arrange
+    check = ClassPerformance(class_list_to_show=[1]).add_condition_test_performance_not_less_than(0.5)
+
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
+                       device=device)
+
+    assert_that(result.conditions_results[0].is_pass, is_(True))
+
+
+def test_condition_test_performance_not_less_than_fail(mnist_dataset_train,
+                                                       mnist_dataset_test,
+                                                       mock_trained_mnist,
+                                                       device):
+    # Arrange
+    check = ClassPerformance(n_to_show=2,
+                             alternative_metrics={'p': Precision(), 'r': Recall()})\
+        .add_condition_test_performance_not_less_than(1)
+
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
+                       device=device)
+
+    assert_that(result.conditions_results[0].is_pass, is_(False))
+
+
+def test_condition_train_test_relative_degradation_not_greater_than_pass(mnist_dataset_train,
+                                                       mnist_dataset_test,
+                                                       mock_trained_mnist,
+                                                       device):
+    # Arrange
+    check = ClassPerformance(class_list_to_show=[1]).add_condition_train_test_relative_degradation_not_greater_than(0.1)
+
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
+                       device=device)
+
+    assert_that(result.conditions_results[0].is_pass, is_(True))
+
+
+def test_condition_train_test_relative_degradation_not_greater_than_fail(mnist_dataset_train,
+                                                       mnist_dataset_test,
+                                                       mock_trained_mnist,
+                                                       device):
+    # Arrange
+    check = ClassPerformance()\
+        .add_condition_train_test_relative_degradation_not_greater_than(0)
+
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
+                       device=device)
+
+    assert_that(result.conditions_results[0].is_pass, is_(False))
+
+
+def test_condition_class_performance_imbalance_ratio_not_greater_than(mnist_dataset_train,
+                                                                      mnist_dataset_test,
+                                                                      mock_trained_mnist,
+                                                                      device):
+    # Arrange
+    check = ClassPerformance()\
+        .add_condition_class_performance_imbalance_ratio_not_greater_than(0.5, 'Precision')
+
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
+                       device=device)
+    print(result.value)
+    print(result.conditions_results)
+
+    assert_that(result.conditions_results[0].is_pass, is_(True))
+
+
+def test_condition_class_performance_imbalance_ratio_not_greater_than_fail(mnist_dataset_train,
+                                                                           mnist_dataset_test,
+                                                                           mock_trained_mnist,
+                                                                           device):
+    # Arrange
+    check = ClassPerformance()\
+        .add_condition_class_performance_imbalance_ratio_not_greater_than(0, 'Precision')
+
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
+                       device=device)
+
+    assert_that(result.conditions_results[0].is_pass, is_(False))
+
+